<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:type" content="website">
  <meta property="og:title" content="静觅">
  <meta property="og:url" content="https://cuiqingcai.com/page/21/index.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:locale" content="zh_CN">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <link rel="canonical" href="https://cuiqingcai.com/page/21/">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: true,
      isPost: false,
      lang: 'zh-CN'
    };

  </script>
  <title>静觅丨崔庆才的个人站点</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content index posts-expand">
            <div class="carousel">
              <div id="wowslider-container">
                <div class="ws_images">
                  <ul>
                    <li><a target="_blank" href="https://cuiqingcai.com/5052.html"><img title="Python3网络爬虫开发实战教程" src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a></li>
                    <li><a target="_blank" href="https://t.lagou.com/fRCBRsRCSN6FA"><img title="52讲轻松搞定网络爬虫" src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a></li>
                    <li><a target="_blank" href="https://brightdata.grsm.io/cuiqingcai"><img title="亮网络解锁器" src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/4320.html"><img title="Python3网络爬虫开发视频教程" src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/5094.html"><img title="爬虫代理哪家强？十大付费代理详细对比评测出炉！" src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a></li>
                  </ul>
                </div>
                <div class="ws_thumbs">
                  <div>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a>
                  </div>
                </div>
                <div class="ws_shadow"></div>
              </div>
            </div>
            <link rel="stylesheet" href="/lib/wowslide/slide.css">
            <script src="/lib/wowslide/jquery.min.js"></script>
            <script src="/lib/wowslide/slider.js"></script>
            <script>
              jQuery("#wowslider-container").wowSlider(
              {
                effect: "cube",
                prev: "",
                next: "",
                duration: 20 * 100,
                delay: 20 * 100,
                width: 716,
                height: 297,
                autoPlay: true,
                playPause: true,
                stopOnHover: false,
                loop: false,
                bullets: 0,
                caption: true,
                captionEffect: "slide",
                controls: true,
                onBeforeStep: 0,
                images: 0
              });

            </script>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4925.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4925.html" class="post-title-link" itemprop="url">TensorFlow RNN Cell源码解析</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p> 本文介绍下 RNN 及几种变种的结构和对应的 TensorFlow 源码实现，另外通过简单的实例来实现 TensorFlow RNN 相关类的调用。</p>
                  <h2 id="RNN"><a href="#RNN" class="headerlink" title="RNN"></a>RNN</h2>
                  <p>RNN，循环神经网络，Recurrent Neural Networks。人们思考问题往往不是从零开始的，比如阅读时我们对每个词的理解都会依赖于前面看到的一些信息，而不是把前面看的内容全部抛弃再去理解某处的信息。应用到深度学习上面，如果我们想要学习去理解一些依赖上文的信息，RNN 便可以做到，它有一个循环的操作，可以使其可以保留之前学习到的内容。 RNN 的结构如下： <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-27-22-35-41.jpg" alt=""> 在上图网络结构中，对于矩形块 A 的那部分，通过输入xt（t时刻的特征向量），它会输出一个结果ht（t时刻的状态或者输出）。网络中的循环结构使得某个时刻的状态能够传到下一个时刻。 这些循环的结构让 RNNs 看起来有些难以理解，但我们可以把 RNNs 看成是一个普通的网络做了多次复制后叠加在一起组成的，每一网络会把它的输出传递到下一个网络中。我们可以把 RNNs 在时间步上进行展开，就得到下图这样： <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-27-22-37-24.jpg" alt=""> 所以最基本的 RNN Cell 输入就是 xt，它还会输出一个隐含内容传递到下一个 Cell，同时还会生成一个结果 ht，其最基本的结构如如下： <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-27-23-59-19.jpg" alt=""> 仅仅是输入的 xt 和隐藏状态进行 concat，然后经过线性变换后经过一个 tanh 激活函数便输出了，另外隐含内容和输出结果是相同的内容。 我们来分析一下 TensorFlow 里面 RNN Cell 的实现。 TensorFlow 实现 RNN Cell 的位置在 python/ops/rnn<em>cell<em>impl.py，首先其实现了一个 RNNCell 类，继承了 Layer 类，其内部有三个比较重要的方法，state_size()、output_size()、__call</em></em>() 方法，其中 state_size() 和 output_size() 方法设置为类属性，可以当做属性来调用，实现如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">@property</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">state_size</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""size(s) of state(s) used by this cell.</span></span><br><span class="line"><span class="string">It can be represented by an Integer, a TensorShape or a tuple of Integers</span></span><br><span class="line"><span class="string">or TensorShapes.</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line">    <span class="keyword">raise</span> NotImplementedError(<span class="string">"Abstract method"</span>)</span><br><span class="line"></span><br><span class="line"><span class="meta">@property</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">output_size</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""Integer or TensorShape: size of outputs produced by this cell."""</span></span><br><span class="line">    <span class="keyword">raise</span> NotImplementedError(<span class="string">"Abstract method"</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>分别代表 Cell 的状态和输出维度，和 Cell 中的神经元数量有关，但这里两个方法都没有实现，意思是说我们必须要实现一个子类继承 RNNCell 类并实现这两个方法。 另外对于 <strong>call</strong>() 方法，实际上就是当初始化的对象直接被调用的时候触发的方法，实现如下：</p>
                  <figure class="highlight pf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def __call__(<span class="literal">self</span>, inputs, <span class="keyword">state</span>, scope=None):</span><br><span class="line">    if scope is not None:</span><br><span class="line">        with vs.variable_scope(scope,</span><br><span class="line">                               custom_getter=<span class="literal">self</span>._rnn_get_variable) as scope:</span><br><span class="line">            return super(RNNCell, <span class="literal">self</span>).__call__(inputs, <span class="keyword">state</span>, scope=scope)</span><br><span class="line">    else:</span><br><span class="line">        with vs.variable_scope(vs.get_variable_scope(),</span><br><span class="line">                               custom_getter=<span class="literal">self</span>._rnn_get_variable):</span><br><span class="line">            return super(RNNCell, <span class="literal">self</span>).__call__(inputs, <span class="keyword">state</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>实际上是调用了父类 Layer 的 <strong>call</strong>() 方法，但父类中 <strong>call</strong>() 方法中又调用了 call() 方法，而 Layer 类的 call() 方法的实现如下：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">call</span><span class="params">(<span class="keyword">self</span>, inputs, **kwargs)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">return</span> inputs</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>父类的 call() 方法实现非常简单，所以要实现其真正的功能，只需要在继承 RNNCell 类的子类中实现 call() 方法即可。 接下来我们看下 RNN Cell 的最基本的实现，叫做 BasicRNNCell，其代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">BasicRNNCell</span><span class="params">(RNNCell)</span>:</span></span><br><span class="line">  <span class="string">"""The most basic RNN cell.</span></span><br><span class="line"><span class="string">  Args:</span></span><br><span class="line"><span class="string">    num_units: int, The number of units in the RNN cell.</span></span><br><span class="line"><span class="string">    activation: Nonlinearity to use.  Default: `tanh`.</span></span><br><span class="line"><span class="string">    reuse: (optional) Python boolean describing whether to reuse variables</span></span><br><span class="line"><span class="string">     in an existing scope.  If not `True`, and the existing scope already has</span></span><br><span class="line"><span class="string">     the given variables, an error is raised.</span></span><br><span class="line"><span class="string">  """</span></span><br><span class="line"></span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, num_units, activation=None, reuse=None)</span>:</span></span><br><span class="line">    super(BasicRNNCell, self).__init__(_reuse=reuse)</span><br><span class="line">    self._num_units = num_units</span><br><span class="line">    self._activation = activation <span class="keyword">or</span> math_ops.tanh</span><br><span class="line">    self._linear = <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"><span class="meta">  @property</span></span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">state_size</span><span class="params">(self)</span>:</span></span><br><span class="line">    <span class="keyword">return</span> self._num_units</span><br><span class="line"></span><br><span class="line"><span class="meta">  @property</span></span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">output_size</span><span class="params">(self)</span>:</span></span><br><span class="line">    <span class="keyword">return</span> self._num_units</span><br><span class="line"></span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">call</span><span class="params">(self, inputs, state)</span>:</span></span><br><span class="line">    <span class="string">"""Most basic RNN: output = new_state = act(W * input + U * state + B)."""</span></span><br><span class="line">    <span class="keyword">if</span> self._linear <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">      self._linear = _Linear([inputs, state], self._num_units, <span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">    output = self._activation(self._linear([inputs, state]))</span><br><span class="line">    <span class="keyword">return</span> output, output</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到在初始化的时候，最终要的一个参数是 num<em>units，意思就是这个 Cell 中神经元的个数，另外还有一个参数 activation 即默认使用的激活函数，默认使用的 tanh，reuse 代表该 Cell 是否可以被重新使用。 在 state<em>size()、output_size() 方法里，其返回的内容都是 num_units，即神经元的个数，接下来 call() 方法中，传入的参数为 inputs 和 state，即输入的 x 和 上一次的隐含状态，首先实例化了一个 _Linear 类，这个类实际上就是做线性变换的类，将二者传递过来，然后直接调用，就实现了 w * [inputs, state] + b 的线性变换，其中 _Linear 类的 __call</em></em>() 方法实现如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def __call__(self, <span class="keyword">args</span>):</span><br><span class="line">    <span class="keyword">if</span> not self._is_sequence:</span><br><span class="line">        <span class="keyword">args</span> = [<span class="keyword">args</span>]</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">len</span>(<span class="keyword">args</span>) == <span class="number">1</span>:</span><br><span class="line">        <span class="keyword">res</span> = math_ops.matmul(<span class="keyword">args</span>[<span class="number">0</span>], self._weights)</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        <span class="keyword">res</span> = math_ops.matmul(array_ops.concat(<span class="keyword">args</span>, <span class="number">1</span>), self._weights)</span><br><span class="line">    <span class="keyword">if</span> self._build_bia<span class="variable">s:</span></span><br><span class="line">        <span class="keyword">res</span> = nn_ops.bias_add(<span class="keyword">res</span>, self._biases)</span><br><span class="line">    <span class="keyword">return</span> <span class="keyword">res</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>很明显这里传递了 [inputs, state] 作为 <strong>call</strong>() 方法的 args，会执行 concat() 和 matmul() 方法，然后接着再执行 bias_add() 方法，这样就实现了线性变换。 最后回到 BasicRNNCell 的 call() 方法中，在 _linear() 方法外面又包括了一层 _activation() 方法，即对线性变换应用一次 tanh 激活函数处理，作为输出结果。 最后返回的结果是 output 和 output，第一个代表 output，第二个代表隐状态，其值也等于 output。 我们用一个实例来感受一下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import tensorflow as tf</span><br><span class="line"></span><br><span class="line">cell = tf<span class="selector-class">.nn</span><span class="selector-class">.rnn_cell</span>.BasicRNNCell(num_units=<span class="number">128</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(cell.state_size)</span></span></span><br><span class="line">inputs = tf.placeholder(tf<span class="selector-class">.float32</span>, shape=[<span class="number">32</span>, <span class="number">100</span>])</span><br><span class="line">h0 = cell.zero_state(<span class="number">32</span>, tf.float32)</span><br><span class="line">output, <span class="selector-tag">h1</span> = cell(inputs=inputs, state=h0)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(output, output.shape)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(h1, h1.shape)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先初始化了一个神经元个数为 128 的 BasicRNNCell 类，然后构造了一个 shape 为 [32, 100] 的变量作为 inputs，其代表 batch_size 为 32, 维度为 100，随后初始化了初始隐藏状态，调用了 zero_state() 方法，然后直接调用 cell，实际上是最终调用了其 call() 方法，最后得到 output 和 h1，打印输出结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">128</span></span><br><span class="line">Tensor(<span class="string">"basic_rnn_cell/Tanh:0"</span>, shape=(<span class="number">32</span>, <span class="number">128</span>), dtype=<span class="built_in">float</span>32) (<span class="number">32</span>, <span class="number">128</span>)</span><br><span class="line">Tensor(<span class="string">"basic_rnn_cell/Tanh:0"</span>, shape=(<span class="number">32</span>, <span class="number">128</span>), dtype=<span class="built_in">float</span>32) (<span class="number">32</span>, <span class="number">128</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，当输入变量维度为 100 的时候，经过一个 128 神经元 Cell 之后，输出维度变成了 128，其输出 shape 变成了 [32, 128]，且此时输出结果和隐藏状态是相同的。</p>
                  <h2 id="LSTM"><a href="#LSTM" class="headerlink" title="LSTM"></a>LSTM</h2>
                  <p>RNNs 的出现，主要是因为它们能够把以前的信息联系到现在，从而解决现在的问题。比如，利用前面的信息，能够帮助我们理解当前的内容。 有时候，我们在处理当前任务的时候，只需要看一下比较近的一些信息。比如在一个语言模型中，我们要通过上文来预测一下个词可能是什么，那么当我们看到 “the clouds are in the?”时，不需要更多的信息，我们就能够自然而然的想到下一个词应该是“sky”。在这样的情况下，我们所要预测的内容和相关信息之间的间隔很小，这种情况下 RNNs 就能够利用过去的信息， 很容易实现： <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-27-23-55-28.jpg" alt=""> 但是如果我们想依赖前文距离非常远的信息时，普通的 RNN 就非常难以做到了，随着间隔信息的增大，RNN 难以对其做关联： <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-27-23-57-15.jpg" alt=""> 但是 LSTM 可以用来解决这个问题。 LSTM，Long Short Term Memory Networks，是 RNN 的一个变种，经试验它可以用来解决更多问题，并取得了非常好的效果。 LSTM Cell 的结构如下： <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-28-00-01-01.jpg" alt=""> LSTMs 最关键的地方在于 Cell 的状态 和 结构图上面的那条横穿的水平线。 Cell 状态的传输就像一条传送带，向量从整个 Cell 中穿过，只是做了少量的线性操作。这种结构能够很轻松地实现信息从整个 Cell 中穿过而不做改变。 <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-28-00-02-30.jpg" alt=""> 若只有上面的那条水平线是没办法实现添加或者删除信息的，信息的操作是是通过一种叫做门的结构来实现的。 这里我们可以把门分为三个：遗忘门（Forget Gate）、传入门（Input Gate）、输出门（Output Gate）。</p>
                  <h3 id="遗忘门（Forget-Gate）"><a href="#遗忘门（Forget-Gate）" class="headerlink" title="遗忘门（Forget Gate）"></a>遗忘门（Forget Gate）</h3>
                  <p>首先是 LSTM 要决定让那些信息继续通过这个 Cell，这是通过 Forget Gate 的 sigmoid 神经层来实现的。它的输入是ht−1和xt，输出是一个数值都在 0，1 之间的向量，表示让 Ct−1 的各部分信息通过的比重。 0 表示“不让任何信息通过”， 1 表示“让所有信息通过”。 <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-28-00-07-04.jpg" alt=""></p>
                  <h3 id="传入门（Input-Gate）"><a href="#传入门（Input-Gate）" class="headerlink" title="传入门（Input Gate）"></a>传入门（Input Gate）</h3>
                  <p>下一步是决定让多少新的信息加入到 Cell 中来，一个叫做 Input Gate 的 sigmoid 层决定哪些信息需要更新，一个 New Input 通过 tanh 生成一个向量，也就是备选的用来更新的内容，Ct~ 。在下一步，我们把这两部分联合起来，对 Cell 的状态进行一个更新。 <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-28-00-10-02.jpg" alt=""> 在经过 Forget Gate 和 Input Gate 处理后，我们就可以对输入的 Ct-1 做更新了，即把Ct−1 更新为 Ct，首先我们把旧的状态 Ct−1 和 ft 相乘， 把一些不想保留的信息忘掉。然后加上 it∗Ct~，这部分信息就是我们要添加的新内容，这样就可以完成对 Ct-1 的更新。 <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-28-00-13-18.jpg" alt=""></p>
                  <h3 id="输出门-（Output-Gate）"><a href="#输出门-（Output-Gate）" class="headerlink" title="输出门 （Output Gate）"></a>输出门 （Output Gate）</h3>
                  <p>最后我们需要来决定输出什么值，输出主要是依赖于 Cell 的状态 Ct，但是又不仅仅依赖于 Ct，而是需要经过一个过滤的处理。首先，我们还是使用一个 sigmoid 层来决定 Ct 中的哪部分信息会被输出。然后我们把 Ct 通过一个 tanh 激活函数处理，然后把其输出和 sigmoid 计算出来的权重相乘，这样就得到了最后输出的结果。 <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-28-00-15-34.jpg" alt=""> 到了最后，其输出结果有三个内容，其中输出结果就是最上面的箭头代指的内容，即最终计算的结果，隐层包括两部分内容，一个是 Ct，一个是最下方的 ht，我们可以将其合并为一个变量来表示。 接下来我们来看下 LSTMCell 的 TensorFlow 代码实现。 首先它的类是 BasicLSTMCell 类，继承了 RNNCell 类，其初始化方法 init() 实现如下：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, num_units, forget_bias=<span class="number">1.0</span>,</span></span></span><br><span class="line"><span class="function"><span class="params">               state_is_tuple=True, activation=None, reuse=None)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">super</span>(BasicLSTMCell, <span class="keyword">self</span>).__init_<span class="number">_</span>(_reuse=reuse)</span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">not</span> <span class="symbol">state_is_tuple:</span></span><br><span class="line">      logging.warn(<span class="string">"%s: Using a concatenated state is slower and will soon be "</span></span><br><span class="line">                   <span class="string">"deprecated.  Use state_is_tuple=True."</span>, <span class="keyword">self</span>)</span><br><span class="line">    <span class="keyword">self</span>._num_units = num_units</span><br><span class="line">    <span class="keyword">self</span>._forget_bias = forget_bias</span><br><span class="line">    <span class="keyword">self</span>._state_is_tuple = state_is_tuple</span><br><span class="line">    <span class="keyword">self</span>._activation = activation <span class="keyword">or</span> math_ops.tanh</span><br><span class="line">    <span class="keyword">self</span>._linear = None</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里必须传入的参数仍然是 num_units，即神经元的个数，然后 forget_bias 是初始化 Forget Gate 的偏置大小，state_is_tuple 指的是输出状态类型是元组类型，activation 代表默认激活函数，reuse 代表是否可以被重复使用。 接下来看下 state_size() 方法和 output_size() 方法，实现如下：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">@property</span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">state_size</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">return</span> (LSTMStateTuple(<span class="keyword">self</span>._num_units, <span class="keyword">self</span>._num_units)</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">self</span>._state_is_tuple <span class="keyword">else</span> <span class="number">2</span> * <span class="keyword">self</span>._num_units)</span><br><span class="line"></span><br><span class="line">@property</span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">output_size</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">return</span> <span class="keyword">self</span>._num_units</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 state_size() 方法变了，因为输出的 state 需要将 Ct 和隐含状态合并，所以它需要包含两部分的内容，如果传入的参数 state_is_tuple 为 True 的话，状态会被表示成一个元组，否则会是 num_units 乘以 2 的数字，默认是元组形式。output_size() 方法则保持不变。 对于 call() 方法，其实现如下：</p>
                  <figure class="highlight nix">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def call(self, inputs, state):</span><br><span class="line">    <span class="string">""</span><span class="string">"Long short-term memory cell (LSTM).</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Args:</span></span><br><span class="line"><span class="string">      inputs: `2-D` tensor with shape `[batch_size x input_size]`.</span></span><br><span class="line"><span class="string">      state: An `LSTMStateTuple` of state tensors, each shaped</span></span><br><span class="line"><span class="string">        `[batch_size x self.state_size]`, if `state_is_tuple` has been set to</span></span><br><span class="line"><span class="string">        `True`.  Otherwise, a `Tensor` shaped</span></span><br><span class="line"><span class="string">        `[batch_size x 2 * self.state_size]`.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Returns:</span></span><br><span class="line"><span class="string">      A pair containing the new hidden state, and the new state (either a</span></span><br><span class="line"><span class="string">        `LSTMStateTuple` or a concatenated state, depending on</span></span><br><span class="line"><span class="string">        `state_is_tuple`).</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    <span class="attr">sigmoid</span> = math_ops.sigmoid</span><br><span class="line">    <span class="comment"># Parameters of gates are concatenated into one multiply for efficiency.</span></span><br><span class="line">    <span class="keyword">if</span> self._state_is_tuple:</span><br><span class="line">        c, <span class="attr">h</span> = state</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        c, <span class="attr">h</span> = array_ops.split(<span class="attr">value=state,</span> <span class="attr">num_or_size_splits=2,</span> <span class="attr">axis=1)</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> self._linear is None:</span><br><span class="line">        self.<span class="attr">_linear</span> = _Linear([inputs, h], <span class="number">4</span> * self._num_units, True)</span><br><span class="line">    <span class="comment"># i = input_gate, j = new_input, f = forget_gate, o = output_gate</span></span><br><span class="line">    i, j, f, <span class="attr">o</span> = array_ops.split(</span><br><span class="line">        <span class="attr">value=self._linear([inputs,</span> h]), <span class="attr">num_or_size_splits=4,</span> <span class="attr">axis=1)</span></span><br><span class="line"></span><br><span class="line">    <span class="attr">new_c</span> = (</span><br><span class="line">        c * sigmoid(f + self._forget_bias) + sigmoid(i) * self._activation(j))</span><br><span class="line">    <span class="attr">new_h</span> = self._activation(new_c) * sigmoid(o)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> self._state_is_tuple:</span><br><span class="line">        <span class="attr">new_state</span> = LSTMStateTuple(new_c, new_h)</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        <span class="attr">new_state</span> = array_ops.concat([new_c, new_h], <span class="number">1</span>)</span><br><span class="line">    return new_h, new_state</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先为了获取 c, h，需要将其从 state 中分离开来，如果传入的 state 是元组的话可以直接分解，否则需要调用 split() 方法来分解：</p>
                  <figure class="highlight pf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">if <span class="literal">self</span>._state_is_tuple:</span><br><span class="line">    c, h = <span class="keyword">state</span></span><br><span class="line">else:</span><br><span class="line">    c, h = array_ops.split(value=<span class="keyword">state</span>, num_or_size_splits=<span class="number">2</span>, axis=<span class="number">1</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来定义了几个门的实现：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">i, j, f, o = array_ops.split(<span class="attribute">value</span>=self._linear([inputs, h]), <span class="attribute">num_or_size_splits</span>=4, <span class="attribute">axis</span>=1)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>放到一起来用 Linear 计算然后分成了 4 份，分别代表 Input Gate、New Input、Forget Gate、Output Gate，用 i、j、f、o 来表示，这时候四个变量都经过了线性变换，乘以权重并做了偏置操作。 接下来就是更新 Ct-1 为 Ct 和得到隐含状态输出了，都是遵循 LSTM 内部的公式实现：</p>
                  <figure class="highlight haxe">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">new</span><span class="type">_c</span> = (c * sigmoid(f + self._forget_bias) + sigmoid(i) * self._activation(j))</span><br><span class="line"><span class="keyword">new</span><span class="type">_h</span> = self._activation(<span class="keyword">new</span><span class="type">_c</span>) * sigmoid(o)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里值得注意的是还多加了一个 _forget_bias 变量，即设置了初始化偏置，以免初始输出为 0 的问题。 最后将 new_c 和 new_h 进行合并，如果要输出元组，那么就合并为元组，否则二者进行 concat 操作，返回的结果是 new_h、new_state，前者即 Cell 的输出结果，后者代表隐含状态：</p>
                  <figure class="highlight haxe">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">if</span> self._state_is_tuple:<span class="type"></span></span><br><span class="line"><span class="type">    new_state </span>= LSTMStateTuple(<span class="keyword">new</span><span class="type">_c</span>, <span class="keyword">new</span><span class="type">_h</span>)</span><br><span class="line"><span class="keyword">else</span>:<span class="type"></span></span><br><span class="line"><span class="type">    new_state </span>= array_ops.concat([<span class="keyword">new</span><span class="type">_c</span>, <span class="keyword">new</span><span class="type">_h</span>], <span class="number">1</span>)</span><br><span class="line"><span class="keyword">return</span> <span class="keyword">new</span><span class="type">_h</span>, <span class="keyword">new</span><span class="type">_state</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们再用一个实例来感受一下 BasicLSTMCell 的用法：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import tensorflow as tf</span><br><span class="line"></span><br><span class="line">cell = tf<span class="selector-class">.nn</span><span class="selector-class">.rnn_cell</span>.BasicLSTMCell(num_units=<span class="number">128</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(cell.state_size)</span></span></span><br><span class="line">inputs = tf.placeholder(tf<span class="selector-class">.float32</span>, shape=(<span class="number">32</span>, <span class="number">100</span>))</span><br><span class="line">h0 = cell.zero_state(<span class="number">32</span>, tf.float32)</span><br><span class="line">output, <span class="selector-tag">h1</span> = cell(inputs=inputs, state=h0)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(h1)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(h1.h, h1.h.shape)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(h1.c, h1.c.shape)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(output, output.shape)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先初始化了一个神经元个数为 128 的 BasicRNNCell 类，然后构造了一个 shape 为 [32, 100] 的变量作为 inputs，其代表 batch_size 为 32, 维度为 100，随后初始化了初始隐藏状态，调用了 zero_state() 方法，然后直接调用 cell，实际上是最终调用了其 call() 方法，最后得到 output 和 h1，此时 h1 是一个元组，它还可以分离成 h 和 c，分别打印其对象和维度，结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">LSTMStateTuple(c=<span class="number">128</span>, h=<span class="number">128</span>)</span><br><span class="line">LSTMStateTuple(c=&lt;tf.Tensor <span class="string">'add_1:0'</span> shape=(<span class="number">32</span>, <span class="number">128</span>) dtype=<span class="built_in">float</span>32&gt;, h=&lt;tf.Tensor <span class="string">'mul_2:0'</span> shape=(<span class="number">32</span>, <span class="number">128</span>) dtype=<span class="built_in">float</span>32&gt;)</span><br><span class="line">Tensor(<span class="string">"mul_2:0"</span>, shape=(<span class="number">32</span>, <span class="number">128</span>), dtype=<span class="built_in">float</span>32) (<span class="number">32</span>, <span class="number">128</span>)</span><br><span class="line">Tensor(<span class="string">"add_1:0"</span>, shape=(<span class="number">32</span>, <span class="number">128</span>), dtype=<span class="built_in">float</span>32) (<span class="number">32</span>, <span class="number">128</span>)</span><br><span class="line">Tensor(<span class="string">"mul_2:0"</span>, shape=(<span class="number">32</span>, <span class="number">128</span>), dtype=<span class="built_in">float</span>32) (<span class="number">32</span>, <span class="number">128</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到其维度都是 [32, 128]，而且 h1.h 和 output 是相同的。 另外 LSTM 有许多变种，其中一个比较有名的就是 Gers &amp; Schmidhuber (2000) 提出的，它在原来的基础上行添加了 Peephole Connections，使得遗忘门可以受 Ct-1 的影响。 <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-28-00-51-47.jpg" alt=""> 另外还有一个变种就是将 Forget Gate 和 Input Gate 二者联合起来，做到要么遗忘老的输入新的，要么保留老的不输入新的。 <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-28-00-54-07.jpg" alt=""> 但接下来还有一个更常用的变种，俺就是 GRU，它是由 Cho, et al. (2014) 提出的，在提出的同时他还提出了 Seq2Seq 模型，为 Generation Model 做好了铺垫。</p>
                  <h3 id="GRU"><a href="#GRU" class="headerlink" title="GRU"></a>GRU</h3>
                  <p>GRU，Gated Recurrent Unit，在 GRU 中，只有两个门：重置门（Reset Gate）和更新门（Update Gate）。同时在这个结构中，把 Ct 和隐藏状态进行了合并，整体结构比标准的 LSTM 结构要简单，而且这个结构后来也非常流行。 <img src="https://germey.gitbooks.io/ai/content/assets/2017-12-28-00-57-08.jpg" alt=""> 接下来我们看下 TensorFlow 中 GRUCell 的实现，代码如下：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">GRUCell</span>(<span class="title">RNNCell</span>):</span></span><br><span class="line">  <span class="string">""</span><span class="string">"Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">  Args:</span></span><br><span class="line"><span class="string">    num_units: int, The number of units in the GRU cell.</span></span><br><span class="line"><span class="string">    activation: Nonlinearity to use.  Default: `tanh`.</span></span><br><span class="line"><span class="string">    reuse: (optional) Python boolean describing whether to reuse variables</span></span><br><span class="line"><span class="string">     in an existing scope.  If not `True`, and the existing scope already has</span></span><br><span class="line"><span class="string">     the given variables, an error is raised.</span></span><br><span class="line"><span class="string">    kernel_initializer: (optional) The initializer to use for the weight and</span></span><br><span class="line"><span class="string">    projection matrices.</span></span><br><span class="line"><span class="string">    bias_initializer: (optional) The initializer to use for the bias.</span></span><br><span class="line"><span class="string">  "</span><span class="string">""</span></span><br><span class="line"></span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>,</span></span></span><br><span class="line"><span class="function"><span class="params">               num_units,</span></span></span><br><span class="line"><span class="function"><span class="params">               activation=None,</span></span></span><br><span class="line"><span class="function"><span class="params">               reuse=None,</span></span></span><br><span class="line"><span class="function"><span class="params">               kernel_initializer=None,</span></span></span><br><span class="line"><span class="function"><span class="params">               bias_initializer=None)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">super</span>(GRUCell, <span class="keyword">self</span>).__init_<span class="number">_</span>(_reuse=reuse)</span><br><span class="line">    <span class="keyword">self</span>._num_units = num_units</span><br><span class="line">    <span class="keyword">self</span>._activation = activation <span class="keyword">or</span> math_ops.tanh</span><br><span class="line">    <span class="keyword">self</span>._kernel_initializer = kernel_initializer</span><br><span class="line">    <span class="keyword">self</span>._bias_initializer = bias_initializer</span><br><span class="line">    <span class="keyword">self</span>._gate_linear = None</span><br><span class="line">    <span class="keyword">self</span>._candidate_linear = None</span><br><span class="line"></span><br><span class="line">  @property</span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">state_size</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">return</span> <span class="keyword">self</span>._num_units</span><br><span class="line"></span><br><span class="line">  @property</span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">output_size</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">return</span> <span class="keyword">self</span>._num_units</span><br><span class="line"></span><br><span class="line">  <span class="function"><span class="keyword">def</span> <span class="title">call</span><span class="params">(<span class="keyword">self</span>, inputs, state)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="string">""</span><span class="string">"Gated recurrent unit (GRU) with nunits cells."</span><span class="string">""</span></span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">self</span>._gate_linear is <span class="symbol">None:</span></span><br><span class="line">      bias_ones = <span class="keyword">self</span>._bias_initializer</span><br><span class="line">      <span class="keyword">if</span> <span class="keyword">self</span>._bias_initializer is <span class="symbol">None:</span></span><br><span class="line">        bias_ones = init_ops.constant_initializer(<span class="number">1.0</span>, dtype=inputs.dtype)</span><br><span class="line">      with vs.variable_scope(<span class="string">"gates"</span>):  <span class="comment"># Reset gate and update gate.</span></span><br><span class="line">        <span class="keyword">self</span>._gate_linear = _Linear(</span><br><span class="line">            [inputs, state],</span><br><span class="line">            <span class="number">2</span> * <span class="keyword">self</span>._num_units,</span><br><span class="line">            True,</span><br><span class="line">            bias_initializer=bias_ones,</span><br><span class="line">            kernel_initializer=<span class="keyword">self</span>._kernel_initializer)</span><br><span class="line"></span><br><span class="line">    value = math_ops.sigmoid(<span class="keyword">self</span>._gate_linear([inputs, state]))</span><br><span class="line">    r, u = array_ops.split(value=value, num_or_size_splits=<span class="number">2</span>, axis=<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">    r_state = r * state</span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">self</span>._candidate_linear is <span class="symbol">None:</span></span><br><span class="line">      with vs.variable_scope(<span class="string">"candidate"</span>)<span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>._candidate_linear = _Linear(</span><br><span class="line">            [inputs, r_state],</span><br><span class="line">            <span class="keyword">self</span>._num_units,</span><br><span class="line">            True,</span><br><span class="line">            bias_initializer=<span class="keyword">self</span>._bias_initializer,</span><br><span class="line">            kernel_initializer=<span class="keyword">self</span>._kernel_initializer)</span><br><span class="line">    c = <span class="keyword">self</span>._activation(<span class="keyword">self</span>._candidate_linear([inputs, r_state]))</span><br><span class="line">    new_h = u * state + (<span class="number">1</span> - u) * c</span><br><span class="line">    <span class="keyword">return</span> new_h, new_h</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在 state_size()、output_size() 方法里，其返回的内容都是 num_units，即神经元的个数。 接下来 call() 方法中，因为 Reset Gate rt 和 Update Gate zt 分别用变量 r、u 表示，它们需要先对 ht-1 即 state 和 xt 做合并，然后再实现线性变换，再调用 sigmod 函数得到：</p>
                  <figure class="highlight pf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">value = math_ops.sigmoid(<span class="literal">self</span>._gate_linear([inputs, <span class="keyword">state</span>]))</span><br><span class="line">r, u = array_ops.split(value=value, num_or_size_splits=<span class="number">2</span>, axis=<span class="number">1</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后需要求解 ht~，首先用 rt 和 ht-1 即 state 相乘：</p>
                  <figure class="highlight pf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">r_state = r * <span class="keyword">state</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后将其放到线性函数里面，在调用 tanh 激活函数即可：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">c = self.<span class="constructor">_activation(<span class="params">self</span>.<span class="params">_candidate_linear</span>([<span class="params">inputs</span>, <span class="params">r_state</span>])</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后计算隐含状态和输出结果，二者一致：</p>
                  <figure class="highlight haxe">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">new</span><span class="type">_h</span> = u * state + (<span class="number">1</span> - u) * c</span><br><span class="line"><span class="keyword">return</span> <span class="keyword">new</span><span class="type">_h</span>, <span class="keyword">new</span><span class="type">_h</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样即可返回得到输出结果和隐藏状态。 我们用一个实例感受一下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import tensorflow as tf</span><br><span class="line"></span><br><span class="line">cell = tf<span class="selector-class">.nn</span><span class="selector-class">.rnn_cell</span>.GRUCell(num_units=<span class="number">128</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(cell.state_size)</span></span></span><br><span class="line">inputs = tf.placeholder(tf<span class="selector-class">.float32</span>, shape=[<span class="number">32</span>, <span class="number">100</span>])</span><br><span class="line">h0 = cell.zero_state(<span class="number">32</span>, tf.float32)</span><br><span class="line">output, <span class="selector-tag">h1</span> = cell(inputs=inputs, state=h0)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(output, output.shape)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(h1, h1.shape)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">128</span></span><br><span class="line">Tensor(<span class="string">"gru_cell/add:0"</span>, shape=(<span class="number">32</span>, <span class="number">128</span>), dtype=<span class="built_in">float</span>32) (<span class="number">32</span>, <span class="number">128</span>)</span><br><span class="line">Tensor(<span class="string">"gru_cell/add:0"</span>, shape=(<span class="number">32</span>, <span class="number">128</span>), dtype=<span class="built_in">float</span>32) (<span class="number">32</span>, <span class="number">128</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个结果和 BasicRNNCell 并无二致，但 GRUCell 内部的结构使模型的效果更加优化，一般我们也会选取 GRUCell 来代替原生的 BasicRNNCell。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>以上便是对 RNN 及一些变种的说明及代码原理分析和实例用法，此部分掌握之后对 Dynamic RNN、多层 RNN 及 RNN Cell 的改写会有很大帮助，需要好好掌握。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-12-28 01:54:02" itemprop="dateCreated datePublished" datetime="2017-12-28T01:54:02+08:00">2017-12-28</time>
                </span>
                <span id="/4925.html" class="post-meta-item leancloud_visitors" data-flag-title="TensorFlow RNN Cell源码解析" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>13k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>12 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4921.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4921.html" class="post-title-link" itemprop="url">TensorFlow MNIST高级学习</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>上一节使用了最简单的网络来处理了 MNIST 数据集，但只有 92% 的正确率，接下来我们使用卷积神经网络来实现更高的正确率。</p>
                  <h2 id="权重初始化"><a href="#权重初始化" class="headerlink" title="权重初始化"></a>权重初始化</h2>
                  <p>在上一节初始化 w 和 b 的时候，我们使用了置零初始化。但在卷积神经网络中，我们需要在初始化的时候权重加入少量噪声来打破对称性和避免零梯度，偏置项直接使用一个较小的正数来避免节点输出恒为零的问题。 所以权重我们可以使用截尾正态分布函数 truncated_normal() 来生成初始化张量，我们可以给它指定均值或标准差，均值默认是 0， 标准差默认是 1，例如我们生成一个 [10] 的张量，代码如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import tensorflow <span class="keyword">as</span> <span class="keyword">tf</span></span><br><span class="line">initial = <span class="keyword">tf</span>.truncated_normal([<span class="number">10</span>], stddev=<span class="number">0.1</span>)</span><br><span class="line">with <span class="keyword">tf</span>.Session() <span class="keyword">as</span> ses<span class="variable">s:</span></span><br><span class="line">    <span class="keyword">print</span>(sess.run(initial))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="number">-0.13058113</span>  <span class="number">0.03201858</span> <span class="number">-0.19349943</span> <span class="number">-0.06061752</span> <span class="number">-0.10267895</span> <span class="number">-0.11079147</span></span><br><span class="line">  <span class="number">0.1881365</span>  <span class="number">-0.01057311</span> <span class="number">-0.02797078</span>  <span class="number">0.01180232</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外 constant() 方法是用于生成常量的方法，例如生成一个 [10] 的常量张量，代码如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import tensorflow <span class="keyword">as</span> <span class="keyword">tf</span></span><br><span class="line">initial = <span class="keyword">tf</span>.constant(<span class="number">0.2</span>, shape=[<span class="number">10</span>])</span><br><span class="line">with <span class="keyword">tf</span>.Session() <span class="keyword">as</span> ses<span class="variable">s:</span></span><br><span class="line">    <span class="keyword">print</span>(sess.run(initial))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[ <span class="number">0.2</span>  <span class="number">0.2</span>  <span class="number">0.2</span>  <span class="number">0.2</span>  <span class="number">0.2</span>  <span class="number">0.2</span>  <span class="number">0.2</span>  <span class="number">0.2</span>  <span class="number">0.2</span>  <span class="number">0.2</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>所以这里我们可以将这两个方法封装成一个函数来尝试：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def weight(shape, <span class="attribute">stddev</span>=0.1, <span class="attribute">mean</span>=0):</span><br><span class="line">    initial = tf.truncated_normal(<span class="attribute">shape</span>=shape, <span class="attribute">mean</span>=mean, <span class="attribute">stddev</span>=stddev)</span><br><span class="line">    return tf.Variable(initial)</span><br><span class="line"></span><br><span class="line">def bias(shape, value):</span><br><span class="line">    initial = tf.constant(<span class="attribute">value</span>=value, <span class="attribute">shape</span>=shape)</span><br><span class="line">    return tf.Variable(initial)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="卷积"><a href="#卷积" class="headerlink" title="卷积"></a>卷积</h2>
                  <p>这次我们需要使用卷积神经网络来处理图片，所以这里的核心部分就是卷积和池化了，首先我们来了解一下卷积和池化。 卷积常用的方法为 conv2d() ，它的 API 如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">tf.nn.conv2d(<span class="keyword">input</span>, <span class="keyword">filter</span>, strides, padding, use_cudnn_on_gpu=<span class="keyword">None</span>, <span class="type">name</span>=<span class="keyword">None</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个方法是 TensorFlow 实现卷积常用的方法，也是搭建卷积神经网络的核心方法，参数介绍如下：</p>
                  <ul>
                    <li>input，指需要做卷积的输入图像，它要求是一个 Tensor，具有 [batch_size, in_height, in_width, in_channels] 这样的 shape，具体含义是 [batch_size 的图片数量, 图片高度, 图片宽度, 输入图像通道数]，注意这是一个 4 维的 Tensor，要求类型为 float32 和 float64 其中之一。</li>
                    <li>filter，相当于 CNN 中的卷积核，它要求是一个 Tensor，具有 [filter_height, filter_width, in_channels, out_channels] 这样的shape，具体含义是 [卷积核的高度，卷积核的宽度，输入图像通道数，输出通道数（即卷积核个数）]，要求类型与参数 input 相同，有一个地方需要注意，第三维 in_channels，就是参数 input 的第四维。</li>
                    <li>strides，卷积时在图像每一维的步长，这是一个一维的向量，长度 4，具有 [stride_batch_size, stride_in_height, stride_in_width, stride_in_channels] 这样的 shape，第一个元素代表在一个样本的特征图上移动，第二三个元素代表在特征图上的高、宽上移动，第四个元素代表在通道上移动。</li>
                    <li>padding，string 类型的量，只能是 SAME、VALID 其中之一，这个值决定了不同的卷积方式。</li>
                    <li>use_cudnn_on_gpu，布尔类型，是否使用 cudnn 加速，默认为true。</li>
                  </ul>
                  <p>返回的结果是 [batch_size, out_height, out_width, out_channels] 维度的结果。 我们这里拿一张 3x3 的图片，单通道（通道为1）的图片，拿一个 1x1 的卷积核进行卷积：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">input = tf.Variable(tf.random_normal([1, 3, 3, 1]))</span><br><span class="line">filter = tf.Variable(tf.random_normal([1, 1, 1, 1]))</span><br><span class="line">op = tf.nn.conv2d(input, filter, strides=[1, 1, 1, 1], <span class="attribute">padding</span>=<span class="string">'VALID'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(op.shape)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="number">1</span>, <span class="number">3</span>, <span class="number">3</span>, <span class="number">1</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>很清晰，一张图片，拿一个 1x1 的核去做卷积，得到的结果输出是 3x3 的，输出通道为 1，batch_size 照旧。 再将卷积核扩大，用一个 3x3 的卷积核：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">input = tf.Variable(tf.random_normal([1, 3, 3, 1]))</span><br><span class="line">filter = tf.Variable(tf.random_normal([3, 3, 1, 1]))</span><br><span class="line">op = tf.nn.conv2d(input, filter, strides=[1, 1, 1, 1], <span class="attribute">padding</span>=<span class="string">'VALID'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(op.shape)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后输出的是一个 1x1 的值。 将图片扩大为 7x7，卷积核仍然使用 3x3：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">input = tf.Variable(tf.random_normal([1, 7, 7, 1]))</span><br><span class="line">filter = tf.Variable(tf.random_normal([3, 3, 1, 1]))</span><br><span class="line">op = tf.nn.conv2d(input, filter, strides=[1, 1, 1, 1], <span class="attribute">padding</span>=<span class="string">'VALID'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(op.shape)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="number">1</span>, <span class="number">5</span>, <span class="number">5</span>, <span class="number">1</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后输出的是一个 5x5 的值。 这时如果我们把 padding 模式改为 SAME，表示卷积核可以停留在图像边缘：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">input = tf.Variable(tf.random_normal([1, 7, 7, 1]))</span><br><span class="line">filter = tf.Variable(tf.random_normal([3, 3, 1, 1]))</span><br><span class="line">op = tf.nn.conv2d(input, filter, strides=[1, 1, 1, 1], <span class="attribute">padding</span>=<span class="string">'SAME'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(op.shape)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="number">1</span>, <span class="number">7</span>, <span class="number">7</span>, <span class="number">1</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>则输出的内容和原图像是相同的。 这时如果更改 batch_size 和 out_channels，比如 batch_size 修改为 3，out_channels 修改为 6：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">input = tf.Variable(tf.random_normal([3, 7, 7, 1]))</span><br><span class="line">filter = tf.Variable(tf.random_normal([3, 3, 1, 6]))</span><br><span class="line">op = tf.nn.conv2d(input, filter, strides=[1, 1, 1, 1], <span class="attribute">padding</span>=<span class="string">'SAME'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(op.shape)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="number">3</span>, <span class="number">7</span>, <span class="number">7</span>, <span class="number">6</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出结果的 batch_size 和 out_channels 会随之变化。 当 strides 的步长不为 1 的时候，我们将 stride_in_height 和 stride_in_width 修改为 2，相当于每次移动两步：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">input = tf.Variable(tf.random_normal([3, 7, 7, 1]))</span><br><span class="line">filter = tf.Variable(tf.random_normal([3, 3, 1, 6]))</span><br><span class="line">op = tf.nn.conv2d(input, filter, strides=[1, 2, 2, 1], <span class="attribute">padding</span>=<span class="string">'VALID'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(op.shape)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="number">3</span>, <span class="number">3</span>, <span class="number">3</span>, <span class="number">6</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后我们用一个例子来感受一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import tensorflow as tf</span><br><span class="line"></span><br><span class="line">input = tf.Variable(tf.random_normal([2, 4, 4, 5]))</span><br><span class="line">filter = tf.Variable(tf.random_normal([2, 2, 5, 2]))</span><br><span class="line">op = tf.nn.conv2d(input, filter, strides=[1, 1, 1, 1], <span class="attribute">padding</span>=<span class="string">'VALID'</span>)</span><br><span class="line">sess = tf.InteractiveSession()</span><br><span class="line">tf.global_variables_initializer().<span class="builtin-name">run</span>()</span><br><span class="line"><span class="builtin-name">print</span>(op.shape)</span><br><span class="line"><span class="builtin-name">print</span>(sess.<span class="builtin-name">run</span>(op))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 input、filter 通过指定 shape 的方式调用 random_normal() 方法进行随机初始化，input 的维度为 [2, 4, 4, 5]，即 batch_size 为 2，图片是 4x4，输入通道数为 5，卷积核大小为 2x2，输入通道 5，输出通道 2，步长为 1，padding 方式选用 VALID，最后输出得到输出的 shape 和结果。 结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="number">2</span>, <span class="number">3</span>, <span class="number">3</span>, <span class="number">2</span>)</span><br><span class="line">[[[[  <span class="number">2.05039382</span>  <span class="number">-8.82934952</span>]</span><br><span class="line">   [ <span class="number">-9.77668381</span>   <span class="number">3.63882256</span>]</span><br><span class="line">   [ <span class="number">-4.46390772</span>  <span class="number">-5.91670704</span>]]</span><br><span class="line"></span><br><span class="line">  [[  <span class="number">8.41201782</span>  <span class="number">-6.72245312</span>]</span><br><span class="line">   [ <span class="number">-1.47592044</span>  <span class="number">13.03628349</span>]</span><br><span class="line">   [  <span class="number">5.44015312</span>   <span class="number">2.46059227</span>]]</span><br><span class="line"></span><br><span class="line">  [[ <span class="number">-3.18967772</span>   <span class="number">1.24733043</span>]</span><br><span class="line">   [<span class="number">-10.1108532</span>   <span class="number">-6.44734669</span>]</span><br><span class="line">   [  <span class="number">1.99426246</span>   <span class="number">2.91549349</span>]]]</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> [[[ <span class="number">-1.66685319</span>   <span class="number">0.32011557</span>]</span><br><span class="line">   [ <span class="number">-5.66163826</span>  <span class="number">-0.37670898</span>]</span><br><span class="line">   [ <span class="number">-0.74658942</span>   <span class="number">1.31723833</span>]]</span><br><span class="line"></span><br><span class="line">  [[ <span class="number">-5.85412216</span>  <span class="number">-0.29930949</span>]</span><br><span class="line">   [ <span class="number">-0.75974303</span>  <span class="number">-1.84006214</span>]</span><br><span class="line">   [ <span class="number">-2.05475235</span>   <span class="number">4.9572196</span> ]]</span><br><span class="line"></span><br><span class="line">  [[ <span class="number">-4.09344864</span>   <span class="number">1.39405775</span>]</span><br><span class="line">   [ <span class="number">-1.28887582</span>  <span class="number">-2.82365012</span>]</span><br><span class="line">   [  <span class="number">4.87360907</span>  <span class="number">10.8071022</span> ]]]]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到 input 维度为 [2, 4, 4, 5]，filter 维度为 [2, 2, 5, 2] 时，生成的结果维度为 [2, 3, 3, 2]。</p>
                  <h2 id="池化"><a href="#池化" class="headerlink" title="池化"></a>池化</h2>
                  <p>池化层往往在卷积层后面，通过池化来降低卷积层输出的特征向量，同时改善结果。 在这里介绍一个常用的最大值池化 max_pool() 方法，其 API 如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">tf.nn.max<span class="constructor">_pool(<span class="params">value</span>, <span class="params">ksize</span>, <span class="params">strides</span>, <span class="params">padding</span>, <span class="params">name</span>=None)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>是CNN当中的最大值池化操作，其实用法和卷积很类似。 参数介绍如下：</p>
                  <ul>
                    <li>value，需要池化的输入，一般池化层接在卷积层后面，所以输入通常是 feature map，依然是 [batch_size, height, width, channels] 这样的shape。</li>
                    <li>ksize，池化窗口的大小，取一个四维向量，一般是 [batch_size, height, width, channels]，因为我们不想在 batch 和 channels 上做池化，所以这两个维度设为了1。</li>
                    <li>strides，和卷积类似，窗口在每一个维度上滑动的步长，一般也是 [stride_batch_size, stride_height, stride_width, stride_channels]。</li>
                    <li>padding，和卷积类似，可以取 VALID、SAME，返回一个 Tensor，类型不变，shape 仍然是 [batch_size, height, width, channels] 这种形式。</li>
                  </ul>
                  <p>在这里输入为 [3, 7, 7, 2]，池化窗口设置为 [1, 2, 2, 1]，步长为 [1, 1, 1, 1]，padding 模式设置为 VALID。</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">input = tf.Variable(tf.random_normal([<span class="number">3</span>, <span class="number">7</span>, <span class="number">7</span>, <span class="number">2</span>]))</span><br><span class="line">op = tf.nn.max_pool(input, ksize=[<span class="number">1</span>, <span class="number">2</span>, <span class="number">2</span>, <span class="number">1</span>], strides=[<span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>], padding=<span class="string">'VALID'</span>)</span><br><span class="line">print(op.shape)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="number">3</span>, <span class="number">6</span>, <span class="number">6</span>, <span class="number">2</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>类似的原理，我们可以得到这样的的结果。</p>
                  <h2 id="卷积和池化"><a href="#卷积和池化" class="headerlink" title="卷积和池化"></a>卷积和池化</h2>
                  <p>所以了解了以上卷积和池化方法的用法，我们可以定义如下两个工具方法：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def conv2d(input, filter, strides=[1, 1, 1, 1], <span class="attribute">padding</span>=<span class="string">'SAME'</span>):</span><br><span class="line">    return tf.nn.conv2d(input, filter, <span class="attribute">strides</span>=strides, <span class="attribute">padding</span>=padding)</span><br><span class="line"></span><br><span class="line">def max_pool(input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], <span class="attribute">padding</span>=<span class="string">'SAME'</span>):</span><br><span class="line">    return tf.nn.max_pool(input, <span class="attribute">ksize</span>=ksize, <span class="attribute">strides</span>=strides, <span class="attribute">padding</span>=padding)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这两个方法分别实现了卷积和池化，并设置了默认步长和核大小。 接下来就让我们开始神经网络的构建吧。</p>
                  <h2 id="初始化"><a href="#初始化" class="headerlink" title="初始化"></a>初始化</h2>
                  <p>首先我们需要初始化一些数据，包括输入的 x 和对一个的标注 y_label：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.placeholder(tf.<span class="built_in">float</span>32, shape=[None, <span class="number">784</span>])</span><br><span class="line">y_label = tf.placeholder(tf.<span class="built_in">float</span>32, shape=[None, <span class="number">10</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="第一层卷积"><a href="#第一层卷积" class="headerlink" title="第一层卷积"></a>第一层卷积</h2>
                  <p>现在我们可以开始实现第一层了。它由一个卷积接一个 max pooling 完成。卷积在每个 5x5 的 patch 中算出 32 个特征。卷积的权重张量形状是 [5, 5, 1, 32]，前两个维度是 patch 的大小，接着是输入的通道数目，最后是输出的通道数目，而对于每一个输出通道都有一个对应的偏置量，我们首先初始化 w 和 b</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">w_conv1 = weight([<span class="number">5</span>, <span class="number">5</span>, <span class="number">1</span>, <span class="number">32</span>])</span><br><span class="line">b_conv1 = bias([<span class="number">32</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>为了用这一层，我们把 x 变成一个四维向量，其第 2、3 维对应图片的宽、高，最后一维代表图片的颜色通道数，因为是灰度图所以这里的通道数为 1，如果是彩色图，则为 3。 随后我们需要对图片做 reshape 操作，将其</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x_reshape = tf.reshape(x, [<span class="number">-1</span>, <span class="number">28</span>, <span class="number">28</span>, <span class="number">1</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们把 x_reshape 和权值向量进行卷积，加上偏置项，然后应用 ReLU 激活函数，最后进行 max pooling：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">h_conv1 = tf.nn.relu(conv2d(x_reshape, w_conv1) + b_conv1)</span><br><span class="line">h_pool1 = max_pool(h_conv1)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="第二层卷积"><a href="#第二层卷积" class="headerlink" title="第二层卷积"></a>第二层卷积</h2>
                  <p>现在我们已经实现了一层卷积，为了构建一个更深的网络，我们再继续增加一层卷积，将通道数变成 64，所以这里的初始化权重和偏置为：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">w_conv2 = weight([<span class="number">5</span>, <span class="number">5</span>, <span class="number">32</span>, <span class="number">64</span>])</span><br><span class="line">b_conv2 = bias([<span class="number">64</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>随后我们把上一层池化结果 h_pool1 和权值向量进行卷积，加上偏置项，然后应用 ReLU 激活函数，最后进行 max pooling：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">h_conv2 = tf.nn.relu(conv2d(h_pool1, w_conv2) + b_conv2)</span><br><span class="line">h_pool2 = max_pool(h_conv2)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="密集连接层"><a href="#密集连接层" class="headerlink" title="密集连接层"></a>密集连接层</h2>
                  <p>现在，图片尺寸减小到7x7，我们再加入一个有 1024 个神经元的全连接层，用于处理整个图片。我们把池化层输出的张量 reshape 成一些向量，乘上权重矩阵，加上偏置，然后对其使用 ReLU。</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">w_fc1 = weight([<span class="number">7</span> * <span class="number">7</span> * <span class="number">64</span>, <span class="number">1024</span>])</span><br><span class="line">b_fc1 = bias([<span class="number">1024</span>])</span><br><span class="line">h_pool2_flat = tf.reshape(h_pool2, [<span class="number">-1</span>, <span class="number">7</span> * <span class="number">7</span> * <span class="number">64</span>])</span><br><span class="line">h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="Dropout"><a href="#Dropout" class="headerlink" title="Dropout"></a>Dropout</h2>
                  <p>为了减少过拟合，我们在输出层之前加入 dropout。我们用一个 placeholder 来代表一个神经元的输出在 dropout 中保持不变的概率。这样我们可以在训练过程中启用 dropout，在测试过程中关闭 dropout。 TensorFlow 的 tf.nn.dropout 操作除了可以屏蔽神经元的输出外，还会自动处理神经元输出值的 scale，所以用 dropout 的时候可以不用考虑 scale。</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">keep_prob</span> = tf.placeholder(tf.float32)</span><br><span class="line"><span class="attr">h_fc1_dropout</span> = tf.nn.dropout(h_fc1, keep_prob=keep_prob)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="输出层"><a href="#输出层" class="headerlink" title="输出层"></a>输出层</h2>
                  <p>最后，我们添加一个 Softmax 输出层，这里我们需要将 1024 维转为 10 维，所以需要声明一个 [1024, 10] 的权重和 [10] 的偏置：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">w_fc2</span> = weight([<span class="number">1024</span>, <span class="number">10</span>])</span><br><span class="line"><span class="attr">b_fc1</span> = bias([<span class="number">10</span>])</span><br><span class="line"><span class="attr">y</span> = tf.nn.softmax(tf.matmul(h_fc1_dropout, w_fc2) + b_fc1)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="训练和评估模型"><a href="#训练和评估模型" class="headerlink" title="训练和评估模型"></a>训练和评估模型</h2>
                  <p>为了进行训练和评估，我们使用与之前简单的单层 Softmax 神经网络模型几乎相同的一套代码，只是我们会用更加复杂的 Adam 优化器来做梯度最速下降，在 feed_dict 中加入额外的参数 keep_prob 来控制 dropout 比例，然后每 100次 迭代输出一次日志：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># Loss</span></span><br><span class="line">cross_entropy = -tf.reduce_sum(y_label * tf.log(y))</span><br><span class="line">train = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Prediction</span></span><br><span class="line">correct_prediction = tf.equal(tf.argmax(y_label, <span class="attribute">axis</span>=1), tf.argmax(y, <span class="attribute">axis</span>=1))</span><br><span class="line">accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))</span><br><span class="line"></span><br><span class="line"><span class="comment"># Train</span></span><br><span class="line">with tf.Session() as sess:</span><br><span class="line">    sess.<span class="builtin-name">run</span>(tf.global_variables_initializer())</span><br><span class="line">    <span class="keyword">for</span> <span class="keyword">step</span> <span class="keyword">in</span> range(total_steps + 1):</span><br><span class="line">        batch = mnist.train.next_batch(batch_size)</span><br><span class="line">        sess.<span class="builtin-name">run</span>(train, feed_dict=&#123;x: batch[0], y_label: batch[1], keep_prob: dropout_keep_prob&#125;)</span><br><span class="line">        # Train accuracy</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">step</span> % steps_per_test == 0:</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'Training Accuracy'</span>, <span class="keyword">step</span>,</span><br><span class="line">                  sess.<span class="builtin-name">run</span>(accuracy, feed_dict=&#123;x: batch[0], y_label: batch[1], keep_prob: 1&#125;))</span><br><span class="line"></span><br><span class="line"><span class="comment"># Final Test</span></span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">'Test Accuracy'</span>, sess.<span class="builtin-name">run</span>(accuracy, feed_dict=&#123;x: mnist.test.images, y_label: mnist.test.labels, keep_prob: 1&#125;))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="运行"><a href="#运行" class="headerlink" title="运行"></a>运行</h2>
                  <p>以上代码，在最终测试集上的准确率大概是99.2%。 运行结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Training Accuracy <span class="number">0</span> <span class="number">0.05</span></span><br><span class="line">Training Accuracy <span class="number">100</span> <span class="number">0.7</span></span><br><span class="line">Training Accuracy <span class="number">200</span> <span class="number">0.85</span></span><br><span class="line">Training Accuracy <span class="number">300</span> <span class="number">0.9</span></span><br><span class="line">Training Accuracy <span class="number">400</span> <span class="number">0.93</span></span><br><span class="line">Training Accuracy <span class="number">500</span> <span class="number">0.91</span></span><br><span class="line">Training Accuracy <span class="number">600</span> <span class="number">0.94</span></span><br><span class="line">Training Accuracy <span class="number">700</span> <span class="number">0.95</span></span><br><span class="line">Training Accuracy <span class="number">800</span> <span class="number">0.95</span></span><br><span class="line">Training Accuracy <span class="number">900</span> <span class="number">0.95</span></span><br><span class="line">Training Accuracy <span class="number">1000</span> <span class="number">0.97</span></span><br><span class="line">Training Accuracy <span class="number">1100</span> <span class="number">0.95</span></span><br><span class="line">Training Accuracy <span class="number">1200</span> <span class="number">0.96</span></span><br><span class="line">Training Accuracy <span class="number">1300</span> <span class="number">0.99</span></span><br><span class="line">Training Accuracy <span class="number">1400</span> <span class="number">0.98</span></span><br><span class="line">Training Accuracy <span class="number">1500</span> <span class="number">0.95</span></span><br><span class="line">Training Accuracy <span class="number">1600</span> <span class="number">0.97</span></span><br><span class="line">Training Accuracy <span class="number">1700</span> <span class="number">1.0</span></span><br><span class="line">Training Accuracy <span class="number">1800</span> <span class="number">0.95</span></span><br><span class="line">Training Accuracy <span class="number">1900</span> <span class="number">0.95</span></span><br><span class="line">Training Accuracy <span class="number">2000</span> <span class="number">0.95</span></span><br><span class="line">Training Accuracy <span class="number">2100</span> <span class="number">0.96</span></span><br><span class="line">Training Accuracy <span class="number">2200</span> <span class="number">0.96</span></span><br><span class="line">Training Accuracy <span class="number">2300</span> <span class="number">0.98</span></span><br><span class="line">Training Accuracy <span class="number">2400</span> <span class="number">0.97</span></span><br><span class="line">Training Accuracy <span class="number">2500</span> <span class="number">0.96</span></span><br><span class="line">Training Accuracy <span class="number">2600</span> <span class="number">0.99</span></span><br><span class="line">Training Accuracy <span class="number">2700</span> <span class="number">0.96</span></span><br><span class="line">Training Accuracy <span class="number">2800</span> <span class="number">0.98</span></span><br><span class="line">Training Accuracy <span class="number">2900</span> <span class="number">0.95</span></span><br><span class="line">Training Accuracy <span class="number">3000</span> <span class="number">0.99</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>本节我们实现了卷积神经网络来处理图像相关问题，将准确率大大提高，可见神经网络是非常强大的。</p>
                  <h2 id="本节代码"><a href="#本节代码" class="headerlink" title="本节代码"></a>本节代码</h2>
                  <p>本节代码地址为：<a href="https://github.com/AIDeepLearning/MNIST" target="_blank" rel="noopener">https://github.com/AIDeepLearning/MNIST</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-12-28 01:51:54" itemprop="dateCreated datePublished" datetime="2017-12-28T01:51:54+08:00">2017-12-28</time>
                </span>
                <span id="/4921.html" class="post-meta-item leancloud_visitors" data-flag-title="TensorFlow MNIST高级学习" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>8.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>8 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4898.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4898.html" class="post-title-link" itemprop="url">TensorFlow MNIST初级学习</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>我们本节要用 MNIST 数据集训练一个可以识别数据的深度学习模型来帮助识别手写数字。</p>
                  <h2 id="MNIST"><a href="#MNIST" class="headerlink" title="MNIST"></a>MNIST</h2>
                  <p>MNIST 是一个入门级计算机视觉数据集，包含了很多手写数字图片，如图所示： <img src="https://germey.gitbooks.io/ai/assets/2017-10-25-14-18-39.png" alt=""> 数据集中包含了图片和对应的标注，在 TensorFlow 中提供了这个数据集，我们可以用如下方法进行导入：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> tensorflow.examples.tutorials.mnist import input_data</span><br><span class="line">mnist = input_data.read_data_sets(<span class="string">'MNIST_data/'</span>, <span class="attribute">one_hot</span>=<span class="literal">True</span>)</span><br><span class="line"><span class="builtin-name">print</span>(mnist)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出结果如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Extracting MNIST_data/train-images-idx3-ubyte.gz</span><br><span class="line">Extracting MNIST_data/train-labels-idx1-ubyte.gz</span><br><span class="line">Extracting MNIST_data/t10k-images-idx3-ubyte.gz</span><br><span class="line">Extracting MNIST_data/t10k-labels-idx1-ubyte.gz</span><br><span class="line"><span class="constructor">Datasets(<span class="params">train</span>=&lt;<span class="params">tensorflow</span>.<span class="params">contrib</span>.<span class="params">learn</span>.<span class="params">python</span>.<span class="params">learn</span>.<span class="params">datasets</span>.<span class="params">mnist</span>.DataSet <span class="params">object</span> <span class="params">at</span> 0x101707ef0&gt;, <span class="params">validation</span>=&lt;<span class="params">tensorflow</span>.<span class="params">contrib</span>.<span class="params">learn</span>.<span class="params">python</span>.<span class="params">learn</span>.<span class="params">datasets</span>.<span class="params">mnist</span>.DataSet <span class="params">object</span> <span class="params">at</span> 0x1016ae4a8&gt;, <span class="params">test</span>=&lt;<span class="params">tensorflow</span>.<span class="params">contrib</span>.<span class="params">learn</span>.<span class="params">python</span>.<span class="params">learn</span>.<span class="params">datasets</span>.<span class="params">mnist</span>.DataSet <span class="params">object</span> <span class="params">at</span> 0x1016f9358&gt;)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里程序会首先下载 MNIST 数据集，然后解压并保存到刚刚制定好的 MNIST_data 文件夹中，然后输出数据集对象。 数据集中包含了 55000 行的训练数据集（mnist.train）、5000 行验证集（mnist.validation）和 10000 行的测试数据集（mnist.test），文件如下所示： <img src="https://germey.gitbooks.io/ai/assets/2017-10-25-14-26-54.jpg" alt=""> 正如前面提到的一样，每一个 MNIST 数据单元有两部分组成：一张包含手写数字的图片和一个对应的标签。我们把这些图片设为 xs，把这些标签设为 ys。训练数据集和测试数据集都包含 xs 和 ys，比如训练数据集的图片是 mnist.train.images ，训练数据集的标签是 mnist.train.labels，每张图片是 28 x 28 像素，即 784 个像素点，我们可以把它展开形成一个向量，即长度为 784 的向量。 所以训练集我们可以转化为 [55000, 784] 的向量，第一维就是训练集中包含的图片个数，第二维是图片的像素点表示的向量。</p>
                  <h2 id="Softmax"><a href="#Softmax" class="headerlink" title="Softmax"></a>Softmax</h2>
                  <p>Softmax 可以看成是一个激励（activation）函数或者链接（link）函数，把我们定义的线性函数的输出转换成我们想要的格式，也就是关于 10 个数字类的概率分布。因此，给定一张图片，它对于每一个数字的吻合度可以被 Softmax 函数转换成为一个概率值。Softmax 函数可以定义为： <img src="https://germey.gitbooks.io/ai/assets/2017-10-25-15-09-37.jpg" alt=""> 展开等式右边的子式，可以得到： <img src="https://germey.gitbooks.io/ai/assets/2017-10-25-15-10-04.jpg" alt=""> 比如判断一张图片中的动物是什么，可能的结果有三种，猫、狗、鸡，假如我们可以经过计算得出它们分别的得分为 3.2、5.1、-1.7，Softmax 的过程首先会对各个值进行次幂计算，分别为 24.5、164.0、0.18，然后计算各个次幂结果占总次幂结果的比重，这样就可以得到 0.13、0.87、0.00 这三个数值，所以这样我们就可以实现差别的放缩，即好的更好、差的更差。 如果要进一步求损失值可以进一步求对数然后取负值，这样 Softmax 后的值如果值越接近 1，那么得到的值越小，即损失越小，如果越远离 1，那么得到的值越大。</p>
                  <h2 id="实现回归模型"><a href="#实现回归模型" class="headerlink" title="实现回归模型"></a>实现回归模型</h2>
                  <p>首先导入 TensorFlow，命令如下：</p>
                  <figure class="highlight elm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> tensorflow <span class="keyword">as</span> tf</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们指定一个输入，在这里输入即为样本数据，如果是训练集那么则是 55000 x 784 的矩阵，如果是验证集则为 5000 x 784 的矩阵，如果是测试集则是 10000 x 784 的矩阵，所以它的行数是不确定的，但是列数是确定的。 所以可以先声明一个 placeholder 对象：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.placeholder(tf.<span class="built_in">float</span>32, [None, <span class="number">784</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里第一个参数指定了矩阵中每个数据的类型，第二个参数指定了数据的维度。 接下来我们需要构建第一层网络，表达式如下： <img src="https://germey.gitbooks.io/ai/assets/2017-10-25-15-27-25.jpg" alt=""> 这里实际上是对输入的 x 乘以 w 权重，然后加上一个偏置项作为输出，而这两个变量实际是在训练的过程中动态调优的，所以我们需要指定它们的类型为 Variable，代码如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">w = tf.<span class="constructor">Variable(<span class="params">tf</span>.<span class="params">zeros</span>([784, 10])</span>)</span><br><span class="line">b = tf.<span class="constructor">Variable(<span class="params">tf</span>.<span class="params">zeros</span>([10])</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来需要实现的就是上图所述的公式了，我们再进一步调用 Softmax 进行计算，得到 y：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">y</span> = <span class="keyword">tf</span>.<span class="keyword">nn</span>.softmax(<span class="keyword">tf</span>.matmul(<span class="keyword">x</span>, <span class="keyword">w</span>) + <span class="keyword">b</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过上面几行代码我们就已经把模型构建完毕了，结构非常简单。</p>
                  <h2 id="损失函数"><a href="#损失函数" class="headerlink" title="损失函数"></a>损失函数</h2>
                  <p>为了训练我们的模型，我们首先需要定义一个指标来评估这个模型是好的。其实，在机器学习，我们通常定义指标来表示一个模型是坏的，这个指标称为成本（cost）或损失（loss），然后尽量最小化这个指标。但是这两种方式是相同的。 一个非常常见的，非常漂亮的成本函数是“交叉熵”（cross-entropy）。交叉熵产生于信息论里面的信息压缩编码技术，但是它后来演变成为从博弈论到机器学习等其他领域里的重要技术手段。它的定义如下： <img src="https://germey.gitbooks.io/ai/assets/2017-10-25-15-45-09.jpg" alt=""> y 是我们预测的概率分布, y_label 是实际的分布，比较粗糙的理解是，交叉熵是用来衡量我们的预测用于描述真相的低效性。 我们可以首先定义 y_label，它的表达式是：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">y_label = tf.placeholder(tf.<span class="built_in">float</span>32, [None, <span class="number">10</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们需要计算它们的交叉熵，代码如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cross_entropy = tf.reduce<span class="constructor">_mean(-<span class="params">tf</span>.<span class="params">reduce_sum</span>(<span class="params">y_label</span> <span class="operator">*</span> <span class="params">tf</span>.<span class="params">log</span>(<span class="params">y</span>)</span>, reduction_indices=<span class="literal">[<span class="number">1</span>]</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先用 reduce_sum() 方法针对每一个维度进行求和，reduction_indices 是指定沿哪些维度进行求和。 然后调用 reduce_mean() 则求平均值，将一个向量中的所有元素求算平均值。 这样我们最后只需要优化这个交叉熵就好了。 所以这样我们再定义一个优化方法：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">train</span> = tf.train.GradientDescentOptimizer(<span class="number">0.5</span>).minimize(cross_entropy)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里使用了 GradientDescentOptimizer，在这里，我们要求 TensorFlow 用梯度下降算法（gradient descent algorithm）以 0.5 的学习速率最小化交叉熵。梯度下降算法（gradient descent algorithm）是一个简单的学习过程，TensorFlow 只需将每个变量一点点地往使成本不断降低的方向移动即可。</p>
                  <h2 id="运行模型"><a href="#运行模型" class="headerlink" title="运行模型"></a>运行模型</h2>
                  <p>定义好了以上内容之后，相当于我们已经构建好了一个计算图，即设置好了模型，我们把它放到 Session 里面运行即可：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">with</span> tf.<span class="constructor">Session()</span> <span class="keyword">as</span> sess:</span><br><span class="line">    sess.run(tf.global<span class="constructor">_variables_initializer()</span>)</span><br><span class="line">    for step <span class="keyword">in</span> range(total_steps + <span class="number">1</span>):</span><br><span class="line">        batch_x, batch_y = mnist.train.next<span class="constructor">_batch(<span class="params">batch_size</span>)</span></span><br><span class="line">        sess.run(train, feed_dict=&#123;x: batch_x, y_label: batch_y&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>该循环的每个步骤中，我们都会随机抓取训练数据中的 batch_size 个批处理数据点，然后我们用这些数据点作为参数替换之前的占位符来运行 train。 这里需要一些变量的定义：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">batch_size</span> = <span class="number">100</span></span><br><span class="line"><span class="attr">total_steps</span> = <span class="number">5000</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="测试模型"><a href="#测试模型" class="headerlink" title="测试模型"></a>测试模型</h2>
                  <p>那么我们的模型性能如何呢？ 首先让我们找出那些预测正确的标签。tf.argmax() 是一个非常有用的函数，它能给出某个 Tensor 对象在某一维上的其数据最大值所在的索引值。由于标签向量是由 0,1 组成，因此最大值 1 所在的索引位置就是类别标签，比如 tf.argmax(y, 1) 返回的是模型对于任一输入 x 预测到的标签值，而 tf.argmax(y_label, 1) 代表正确的标签，我们可以用 tf.equal() 方法来检测我们的预测是否真实标签匹配（索引位置一样表示匹配）。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">correct_prediction = tf.equal(tf.argmax(y, <span class="attribute">axis</span>=1), tf.argmax(y_label, <span class="attribute">axis</span>=1))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这行代码会给我们一组布尔值。为了确定正确预测项的比例，我们可以把布尔值转换成浮点数，然后取平均值。例如，[True, False, True, True] 会变成 [1, 0, 1, 1] ，取平均值后得到 0.75。</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">accuracy = tf.reduce<span class="constructor">_mean(<span class="params">tf</span>.<span class="params">cast</span>(<span class="params">correct_prediction</span>, <span class="params">tf</span>.<span class="params">float32</span>)</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后，我们计算所学习到的模型在测试数据集上面的正确率，定义如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">steps_per_test = 100</span><br><span class="line"><span class="keyword">if</span> <span class="keyword">step</span> % steps_per_test == 0:</span><br><span class="line">    <span class="builtin-name">print</span>(<span class="keyword">step</span>, sess.<span class="builtin-name">run</span>(accuracy, feed_dict=&#123;x: mnist.test.images, y_label: mnist.test.labels&#125;))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个最终结果值应该大约是92%。 这样我们就通过完成了训练和测试阶段，实现了一个基本的训练模型，后面我们会继续优化模型来达到更好的效果。 运行结果如下：</p>
                  <figure class="highlight basic">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">0 </span><span class="number">0.453</span></span><br><span class="line"><span class="symbol">100 </span><span class="number">0.8915</span></span><br><span class="line"><span class="symbol">200 </span><span class="number">0.9026</span></span><br><span class="line"><span class="symbol">300 </span><span class="number">0.9081</span></span><br><span class="line"><span class="symbol">400 </span><span class="number">0.9109</span></span><br><span class="line"><span class="symbol">500 </span><span class="number">0.9108</span></span><br><span class="line"><span class="symbol">600 </span><span class="number">0.9175</span></span><br><span class="line"><span class="symbol">700 </span><span class="number">0.9137</span></span><br><span class="line"><span class="symbol">800 </span><span class="number">0.9158</span></span><br><span class="line"><span class="symbol">900 </span><span class="number">0.9176</span></span><br><span class="line"><span class="symbol">1000 </span><span class="number">0.9167</span></span><br><span class="line"><span class="symbol">1100 </span><span class="number">0.9186</span></span><br><span class="line"><span class="symbol">1200 </span><span class="number">0.9206</span></span><br><span class="line"><span class="symbol">1300 </span><span class="number">0.9161</span></span><br><span class="line"><span class="symbol">1400 </span><span class="number">0.9218</span></span><br><span class="line"><span class="symbol">1500 </span><span class="number">0.9179</span></span><br><span class="line"><span class="symbol">1600 </span><span class="number">0.916</span></span><br><span class="line"><span class="symbol">1700 </span><span class="number">0.9196</span></span><br><span class="line"><span class="symbol">1800 </span><span class="number">0.9222</span></span><br><span class="line"><span class="symbol">1900 </span><span class="number">0.921</span></span><br><span class="line"><span class="symbol">2000 </span><span class="number">0.9223</span></span><br><span class="line"><span class="symbol">2100 </span><span class="number">0.9214</span></span><br><span class="line"><span class="symbol">2200 </span><span class="number">0.9191</span></span><br><span class="line"><span class="symbol">2300 </span><span class="number">0.9228</span></span><br><span class="line"><span class="symbol">2400 </span><span class="number">0.9228</span></span><br><span class="line"><span class="symbol">2500 </span><span class="number">0.9218</span></span><br><span class="line"><span class="symbol">2600 </span><span class="number">0.9197</span></span><br><span class="line"><span class="symbol">2700 </span><span class="number">0.9225</span></span><br><span class="line"><span class="symbol">2800 </span><span class="number">0.9238</span></span><br><span class="line"><span class="symbol">2900 </span><span class="number">0.9219</span></span><br><span class="line"><span class="symbol">3000 </span><span class="number">0.9224</span></span><br><span class="line"><span class="symbol">3100 </span><span class="number">0.9184</span></span><br><span class="line"><span class="symbol">3200 </span><span class="number">0.9253</span></span><br><span class="line"><span class="symbol">3300 </span><span class="number">0.9216</span></span><br><span class="line"><span class="symbol">3400 </span><span class="number">0.9218</span></span><br><span class="line"><span class="symbol">3500 </span><span class="number">0.9212</span></span><br><span class="line"><span class="symbol">3600 </span><span class="number">0.9225</span></span><br><span class="line"><span class="symbol">3700 </span><span class="number">0.9224</span></span><br><span class="line"><span class="symbol">3800 </span><span class="number">0.9225</span></span><br><span class="line"><span class="symbol">3900 </span><span class="number">0.9226</span></span><br><span class="line"><span class="symbol">4000 </span><span class="number">0.9201</span></span><br><span class="line"><span class="symbol">4100 </span><span class="number">0.9138</span></span><br><span class="line"><span class="symbol">4200 </span><span class="number">0.9184</span></span><br><span class="line"><span class="symbol">4300 </span><span class="number">0.9222</span></span><br><span class="line"><span class="symbol">4400 </span><span class="number">0.92</span></span><br><span class="line"><span class="symbol">4500 </span><span class="number">0.924</span></span><br><span class="line"><span class="symbol">4600 </span><span class="number">0.9234</span></span><br><span class="line"><span class="symbol">4700 </span><span class="number">0.9219</span></span><br><span class="line"><span class="symbol">4800 </span><span class="number">0.923</span></span><br><span class="line"><span class="symbol">4900 </span><span class="number">0.9254</span></span><br><span class="line"><span class="symbol">5000 </span><span class="number">0.9218</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>本节通过一个 MNIST 数据集来简单体验了一下真实数据的训练和预测过程，但是准确率还不够高，后面我们会学习用卷积的方式来进行模型训练，准确率会更高。</p>
                  <h2 id="本节代码"><a href="#本节代码" class="headerlink" title="本节代码"></a>本节代码</h2>
                  <p>本节代码地址为：<a href="https://github.com/AIDeepLearning/MNIST" target="_blank" rel="noopener">https://github.com/AIDeepLearning/MNIST</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-12-09 00:36:31" itemprop="dateCreated datePublished" datetime="2017-12-09T00:36:31+08:00">2017-12-09</time>
                </span>
                <span id="/4898.html" class="post-meta-item leancloud_visitors" data-flag-title="TensorFlow MNIST初级学习" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>4.8k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4893.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4893.html" class="post-title-link" itemprop="url">TensorFlow基础入门</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本篇内容基于 Python3 TensorFlow 1.4 版本。</p>
                  <h2 id="本节内容"><a href="#本节内容" class="headerlink" title="本节内容"></a>本节内容</h2>
                  <p>本节通过最简单的示例 —— 平面拟合来说明 TensorFlow 的基本用法。</p>
                  <h2 id="构造数据"><a href="#构造数据" class="headerlink" title="构造数据"></a>构造数据</h2>
                  <p>TensorFlow 的引入方式是：</p>
                  <figure class="highlight elm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> tensorflow <span class="keyword">as</span> tf</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们构造一些随机的三维数据，然后用 TensorFlow 找到平面去拟合它，首先我们用 Numpy 生成随机三维点，其中变量 x 代表三维点的 (x, y) 坐标，是一个 2x100 的矩阵，即 100 个 (x, y)，然后变量 y 代表三位点的 z 坐标，我们用 Numpy 来生成这些随机的点：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> numpy as np</span><br><span class="line">x_data = np.<span class="built_in">float</span>32(np.random.rand(<span class="number">2</span>, <span class="number">100</span>))</span><br><span class="line">y_data = np.dot([<span class="number">0.300</span>, <span class="number">0.200</span>], x_data) + <span class="number">0.400</span></span><br><span class="line"></span><br><span class="line">print(x_data)</span><br><span class="line">print(y_data)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里利用 Numpy 的 random 模块的 rand() 方法生成了 2x100 的随机矩阵，这样就生成了 100 个 (x, y) 坐标，然后用了一个 dot() 方法算了矩阵乘法，用了一个长度为 2 的向量跟此矩阵相乘，得到一个长度为 100 的向量，然后再加上一个常量，得到 z 坐标，输出结果样例如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[[ <span class="number">0.97232962</span>  <span class="number">0.08897641</span>  <span class="number">0.54844421</span>  <span class="number">0.5877986</span>   <span class="number">0.5121088</span>   <span class="number">0.64716059</span></span><br><span class="line">   <span class="number">0.22353953</span>  <span class="number">0.18406206</span>  <span class="number">0.16782761</span>  <span class="number">0.97569454</span>  <span class="number">0.65686035</span>  <span class="number">0.75569868</span></span><br><span class="line">   <span class="number">0.35698661</span>  <span class="number">0.43332314</span>  <span class="number">0.41185728</span>  <span class="number">0.24801297</span>  <span class="number">0.50098598</span>  <span class="number">0.12025958</span></span><br><span class="line">   <span class="number">0.40650111</span>  <span class="number">0.51486945</span>  <span class="number">0.19292323</span>  <span class="number">0.03679928</span>  <span class="number">0.56501174</span>  <span class="number">0.5321334</span></span><br><span class="line">   <span class="number">0.71044683</span>  <span class="number">0.00318134</span>  <span class="number">0.76611853</span>  <span class="number">0.42602748</span>  <span class="number">0.33002195</span>  <span class="number">0.04414672</span></span><br><span class="line">   <span class="number">0.73208278</span>  <span class="number">0.62182301</span>  <span class="number">0.49471655</span>  <span class="number">0.8116194</span>   <span class="number">0.86148429</span>  <span class="number">0.48835048</span></span><br><span class="line">   <span class="number">0.69902027</span>  <span class="number">0.14901569</span>  <span class="number">0.18737803</span>  <span class="number">0.66826463</span>  <span class="number">0.43462989</span>  <span class="number">0.35768151</span></span><br><span class="line">   <span class="number">0.79315376</span>  <span class="number">0.0400687</span>   <span class="number">0.76952982</span>  <span class="number">0.12236254</span>  <span class="number">0.61519378</span>  <span class="number">0.92795062</span></span><br><span class="line">   <span class="number">0.84952474</span>  <span class="number">0.16663995</span>  <span class="number">0.13729768</span>  <span class="number">0.50603199</span>  <span class="number">0.38752931</span>  <span class="number">0.39529857</span></span><br><span class="line">   <span class="number">0.29228279</span>  <span class="number">0.09773371</span>  <span class="number">0.43220878</span>  <span class="number">0.2603009</span>   <span class="number">0.14576958</span>  <span class="number">0.21881725</span></span><br><span class="line">   <span class="number">0.64888018</span>  <span class="number">0.41048348</span>  <span class="number">0.27641159</span>  <span class="number">0.61700606</span>  <span class="number">0.49728736</span>  <span class="number">0.75936913</span></span><br><span class="line">   <span class="number">0.04028837</span>  <span class="number">0.88986284</span>  <span class="number">0.84112513</span>  <span class="number">0.34227493</span>  <span class="number">0.69162005</span>  <span class="number">0.89058989</span></span><br><span class="line">   <span class="number">0.39744586</span>  <span class="number">0.85080278</span>  <span class="number">0.37685293</span>  <span class="number">0.80529863</span>  <span class="number">0.31220895</span>  <span class="number">0.50500977</span></span><br><span class="line">   <span class="number">0.95800418</span>  <span class="number">0.43696108</span>  <span class="number">0.04143282</span>  <span class="number">0.05169986</span>  <span class="number">0.33503434</span>  <span class="number">0.1671818</span></span><br><span class="line">   <span class="number">0.10234453</span>  <span class="number">0.31241918</span>  <span class="number">0.23630807</span>  <span class="number">0.37890589</span>  <span class="number">0.63020509</span>  <span class="number">0.78184551</span></span><br><span class="line">   <span class="number">0.87924582</span>  <span class="number">0.99288088</span>  <span class="number">0.30762389</span>  <span class="number">0.43499199</span>  <span class="number">0.53140771</span>  <span class="number">0.43461791</span></span><br><span class="line">   <span class="number">0.23833922</span>  <span class="number">0.08681628</span>  <span class="number">0.74615192</span>  <span class="number">0.25835371</span>]</span><br><span class="line"> [ <span class="number">0.8174957</span>   <span class="number">0.26717573</span>  <span class="number">0.23811154</span>  <span class="number">0.02851068</span>  <span class="number">0.9627012</span>   <span class="number">0.36802396</span></span><br><span class="line">   <span class="number">0.50543582</span>  <span class="number">0.29964805</span>  <span class="number">0.44869211</span>  <span class="number">0.23191817</span>  <span class="number">0.77344608</span>  <span class="number">0.36636299</span></span><br><span class="line">   <span class="number">0.56170034</span>  <span class="number">0.37465382</span>  <span class="number">0.00471885</span>  <span class="number">0.19509546</span>  <span class="number">0.49715847</span>  <span class="number">0.15201907</span></span><br><span class="line">   <span class="number">0.5642485</span>   <span class="number">0.70218688</span>  <span class="number">0.6031307</span>   <span class="number">0.4705168</span>   <span class="number">0.98698962</span>  <span class="number">0.865367</span></span><br><span class="line">   <span class="number">0.36558965</span>  <span class="number">0.72073907</span>  <span class="number">0.83386165</span>  <span class="number">0.29963031</span>  <span class="number">0.72276717</span>  <span class="number">0.98171854</span></span><br><span class="line">   <span class="number">0.30932376</span>  <span class="number">0.52615297</span>  <span class="number">0.35522953</span>  <span class="number">0.13186514</span>  <span class="number">0.73437029</span>  <span class="number">0.03887378</span></span><br><span class="line">   <span class="number">0.1208882</span>   <span class="number">0.67004597</span>  <span class="number">0.83422536</span>  <span class="number">0.17487818</span>  <span class="number">0.71460873</span>  <span class="number">0.51926661</span></span><br><span class="line">   <span class="number">0.55297899</span>  <span class="number">0.78169805</span>  <span class="number">0.77547258</span>  <span class="number">0.92139858</span>  <span class="number">0.25020468</span>  <span class="number">0.70916855</span></span><br><span class="line">   <span class="number">0.68722379</span>  <span class="number">0.75378138</span>  <span class="number">0.30182058</span>  <span class="number">0.91982585</span>  <span class="number">0.93160367</span>  <span class="number">0.81539184</span></span><br><span class="line">   <span class="number">0.87977934</span>  <span class="number">0.07394848</span>  <span class="number">0.1004181</span>   <span class="number">0.48765802</span>  <span class="number">0.73601437</span>  <span class="number">0.59894943</span></span><br><span class="line">   <span class="number">0.34601998</span>  <span class="number">0.69065076</span>  <span class="number">0.6768015</span>   <span class="number">0.98533565</span>  <span class="number">0.83803362</span>  <span class="number">0.47194552</span></span><br><span class="line">   <span class="number">0.84103006</span>  <span class="number">0.84892255</span>  <span class="number">0.04474261</span>  <span class="number">0.02038293</span>  <span class="number">0.50802571</span>  <span class="number">0.15178065</span></span><br><span class="line">   <span class="number">0.86116213</span>  <span class="number">0.51097614</span>  <span class="number">0.44155359</span>  <span class="number">0.67713588</span>  <span class="number">0.66439205</span>  <span class="number">0.67885226</span></span><br><span class="line">   <span class="number">0.4243969</span>   <span class="number">0.35731083</span>  <span class="number">0.07878648</span>  <span class="number">0.53950399</span>  <span class="number">0.84162414</span>  <span class="number">0.24412845</span></span><br><span class="line">   <span class="number">0.61285144</span>  <span class="number">0.00316137</span>  <span class="number">0.67407191</span>  <span class="number">0.83218956</span>  <span class="number">0.94473189</span>  <span class="number">0.09813353</span></span><br><span class="line">   <span class="number">0.16728765</span>  <span class="number">0.95433819</span>  <span class="number">0.1416636</span>   <span class="number">0.4220584</span>   <span class="number">0.35413414</span>  <span class="number">0.55999744</span></span><br><span class="line">   <span class="number">0.94829601</span>  <span class="number">0.62568033</span>  <span class="number">0.89808714</span>  <span class="number">0.07021013</span>]]</span><br><span class="line">[ <span class="number">0.85519803</span>  <span class="number">0.48012807</span>  <span class="number">0.61215557</span>  <span class="number">0.58204171</span>  <span class="number">0.74617288</span>  <span class="number">0.66775297</span></span><br><span class="line">  <span class="number">0.56814902</span>  <span class="number">0.51514823</span>  <span class="number">0.5400867</span>   <span class="number">0.739092</span>    <span class="number">0.75174732</span>  <span class="number">0.6999822</span></span><br><span class="line">  <span class="number">0.61943605</span>  <span class="number">0.60492771</span>  <span class="number">0.52450095</span>  <span class="number">0.51342299</span>  <span class="number">0.64972749</span>  <span class="number">0.46648169</span></span><br><span class="line">  <span class="number">0.63480003</span>  <span class="number">0.69489821</span>  <span class="number">0.57850311</span>  <span class="number">0.50514314</span>  <span class="number">0.76690145</span>  <span class="number">0.73271342</span></span><br><span class="line">  <span class="number">0.68625198</span>  <span class="number">0.54510222</span>  <span class="number">0.79660789</span>  <span class="number">0.58773431</span>  <span class="number">0.64356002</span>  <span class="number">0.60958773</span></span><br><span class="line">  <span class="number">0.68148959</span>  <span class="number">0.6917775</span>   <span class="number">0.61946087</span>  <span class="number">0.66985885</span>  <span class="number">0.80531934</span>  <span class="number">0.5542799</span></span><br><span class="line">  <span class="number">0.63388372</span>  <span class="number">0.5787139</span>   <span class="number">0.62305848</span>  <span class="number">0.63545502</span>  <span class="number">0.67331071</span>  <span class="number">0.61115777</span></span><br><span class="line">  <span class="number">0.74854193</span>  <span class="number">0.56836022</span>  <span class="number">0.78595346</span>  <span class="number">0.62098848</span>  <span class="number">0.63459907</span>  <span class="number">0.8202189</span></span><br><span class="line">  <span class="number">0.79230218</span>  <span class="number">0.60074826</span>  <span class="number">0.50155342</span>  <span class="number">0.73577477</span>  <span class="number">0.70257953</span>  <span class="number">0.68166794</span></span><br><span class="line">  <span class="number">0.6636407</span>   <span class="number">0.44410981</span>  <span class="number">0.54974625</span>  <span class="number">0.57562188</span>  <span class="number">0.59093375</span>  <span class="number">0.58543506</span></span><br><span class="line">  <span class="number">0.66386805</span>  <span class="number">0.6612752</span>   <span class="number">0.61828378</span>  <span class="number">0.78216895</span>  <span class="number">0.71679293</span>  <span class="number">0.72219985</span></span><br><span class="line">  <span class="number">0.58029252</span>  <span class="number">0.83674336</span>  <span class="number">0.66128606</span>  <span class="number">0.50675907</span>  <span class="number">0.70909116</span>  <span class="number">0.6975331</span></span><br><span class="line">  <span class="number">0.69146618</span>  <span class="number">0.75743606</span>  <span class="number">0.6013666</span>   <span class="number">0.77701676</span>  <span class="number">0.6265411</span>   <span class="number">0.68727338</span></span><br><span class="line">  <span class="number">0.77228063</span>  <span class="number">0.60255049</span>  <span class="number">0.42818714</span>  <span class="number">0.52341076</span>  <span class="number">0.66883513</span>  <span class="number">0.49898023</span></span><br><span class="line">  <span class="number">0.55327365</span>  <span class="number">0.49435803</span>  <span class="number">0.6057068</span>   <span class="number">0.68010968</span>  <span class="number">0.77800791</span>  <span class="number">0.65418036</span></span><br><span class="line">  <span class="number">0.69723127</span>  <span class="number">0.8887319</span>   <span class="number">0.52061989</span>  <span class="number">0.61490928</span>  <span class="number">0.63024914</span>  <span class="number">0.64238486</span></span><br><span class="line">  <span class="number">0.66116097</span>  <span class="number">0.55118095</span>  <span class="number">0.80346301</span>  <span class="number">0.49154814</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就得到了一些三维的点。</p>
                  <h2 id="构造模型"><a href="#构造模型" class="headerlink" title="构造模型"></a>构造模型</h2>
                  <p>随后我们用 TensorFlow 来根据这些数据拟合一个平面，拟合的过程实际上就是寻找 (x, y) 和 z 的关系，即变量 x_data 和变量 y_data 的关系，而它们之间的关系刚才我们用了线性变换表示出来了，即 z = w * (x, y) + b，所以拟合的过程实际上就是找 w 和 b 的过程，所以这里我们就首先像设变量一样来设两个变量 w 和 b，代码如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.placeholder(tf.<span class="built_in">float</span>32, [<span class="number">2</span>, <span class="number">100</span>])</span><br><span class="line">y_label = tf.placeholder(tf.<span class="built_in">float</span>32, [<span class="number">100</span>])</span><br><span class="line">b = tf.Variable(tf.zeros([<span class="number">1</span>]))</span><br><span class="line">w = tf.Variable(tf.random_uniform([<span class="number">2</span>], <span class="number">-1.0</span>, <span class="number">1.0</span>))</span><br><span class="line">y = tf.matmul(tf.reshape(w, [<span class="number">1</span>, <span class="number">2</span>]), x) + b</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在创建模型的时候，我们首先可以将现有的变量来表示出来，用 placeholder() 方法声明即可，一会我们在运行的时候传递给它真实的数据就好，第一个参数是数据类型，第二个参数是形状，因为 x_data 是 2x100 的矩阵，所以这里形状定义为 [2, 100]，而 y_data 是长度为 100 的向量，所以这里形状定义为 [100]，当然此处使用元组定义也可以，不过要写成 (100, )。 随后我们用 Variable 初始化了 TensorFlow 中的变量，b 初始化为一个常量，w 是一个随机初始化的 1x2 的向量，范围在 -1 和 1 之间，然后 y 再用 w、x、b 表示出来，其中 matmul() 方法就是 TensorFlow 中提供的矩阵乘法，类似 Numpy 的 dot() 方法。不过不同的是 matmul() 不支持向量和矩阵相乘，即不能 BroadCast，所以在这里做乘法前需要先调用 reshape() 一下转成 1x2 的标准矩阵，最后将结果表示为 y。 这样我们就构造出来了一个线性模型。 这里的 y 是我们模型中输出的值，而真实的数据却是我们输入的 y_data，即 y_label。</p>
                  <h2 id="损失函数"><a href="#损失函数" class="headerlink" title="损失函数"></a>损失函数</h2>
                  <p>要拟合这个平面的话，我们需要减小 y_label 和 y 的差距就好了，这个差距越小越好。 所以接下来我们可以定义一个损失函数，来代表模型实际输出值和真实值之间的差距，我们的目的就是来减小这个损失，代码实现如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">loss = tf.reduce<span class="constructor">_mean(<span class="params">tf</span>.<span class="params">square</span>(<span class="params">y</span> - <span class="params">y_label</span>)</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里调用了 square() 方法，传入 y_label 和 y 的差来求得平方和，然后使用 reduce_mean() 方法得到这个值的平均值，这就是现在模型的损失值，我们的目的就是减小这个损失值，所以接下来我们使用梯度下降的方法来减小这个损失值即可，定义如下代码：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">optimizer</span> = tf.train.GradientDescentOptimizer(<span class="number">0.5</span>)</span><br><span class="line"><span class="attr">train</span> = optimizer.minimize(loss)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里定义了 GradientDescentOptimizer 优化，即使用梯度下降的方法来减小这个损失值，我们训练模型就是来模拟这个过程。</p>
                  <h2 id="运行模型"><a href="#运行模型" class="headerlink" title="运行模型"></a>运行模型</h2>
                  <p>最后我们将模型运行起来即可，运行时必须声明一个 Session 对象，然后初始化所有的变量，然后执行一步步的训练即可，实现如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">with tf.Session() as sess:</span><br><span class="line">    sess.<span class="builtin-name">run</span>(tf.global_variables_initializer())</span><br><span class="line">    <span class="keyword">for</span> <span class="keyword">step</span> <span class="keyword">in</span> range(201):</span><br><span class="line">        sess.<span class="builtin-name">run</span>(train, feed_dict=&#123;x: x_data, y: y_data&#125;)</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">step</span> % 10 == 0:</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="keyword">step</span>, sess.<span class="builtin-name">run</span>(w), sess.<span class="builtin-name">run</span>(b))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里定义了 200 次循环，每一次循环都会执行一次梯度下降优化，每次循环都调用一次 run() 方法，传入的变量就是刚才定义个 train 对象，feed_dict 就把 placeholder 类型的变量赋值即可。随着训练的进行，损失会越来越小，w 和 b 也会被慢慢调整为拟合的值。 在这里每 10 次 循环我们都打印输出一下拟合的 w 和 b 的值，结果如下：</p>
                  <figure class="highlight basic">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">0 </span>[ <span class="number">0.31494665</span>  <span class="number">0.33602586</span>] [ <span class="number">0.84270978</span>]</span><br><span class="line"><span class="symbol">10 </span>[ <span class="number">0.19601417</span>  <span class="number">0.17301694</span>] [ <span class="number">0.47917289</span>]</span><br><span class="line"><span class="symbol">20 </span>[ <span class="number">0.23550016</span>  <span class="number">0.18053198</span>] [ <span class="number">0.44838765</span>]</span><br><span class="line"><span class="symbol">30 </span>[ <span class="number">0.26029009</span>  <span class="number">0.18700737</span>] [ <span class="number">0.43032286</span>]</span><br><span class="line"><span class="symbol">40 </span>[ <span class="number">0.27547371</span>  <span class="number">0.19152154</span>] [ <span class="number">0.41897511</span>]</span><br><span class="line"><span class="symbol">50 </span>[ <span class="number">0.28481475</span>  <span class="number">0.19454622</span>] [ <span class="number">0.41185945</span>]</span><br><span class="line"><span class="symbol">60 </span>[ <span class="number">0.29058149</span>  <span class="number">0.19652548</span>] [ <span class="number">0.40740564</span>]</span><br><span class="line"><span class="symbol">70 </span>[ <span class="number">0.2941508</span>   <span class="number">0.19780098</span>] [ <span class="number">0.40462157</span>]</span><br><span class="line"><span class="symbol">80 </span>[ <span class="number">0.29636407</span>  <span class="number">0.1986146</span> ] [ <span class="number">0.40288284</span>]</span><br><span class="line"><span class="symbol">90 </span>[ <span class="number">0.29773837</span>  <span class="number">0.19913</span>   ] [ <span class="number">0.40179768</span>]</span><br><span class="line"><span class="symbol">100 </span>[ <span class="number">0.29859257</span>  <span class="number">0.19945487</span>] [ <span class="number">0.40112072</span>]</span><br><span class="line"><span class="symbol">110 </span>[ <span class="number">0.29912385</span>  <span class="number">0.199659</span>  ] [ <span class="number">0.40069857</span>]</span><br><span class="line"><span class="symbol">120 </span>[ <span class="number">0.29945445</span>  <span class="number">0.19978693</span>] [ <span class="number">0.40043539</span>]</span><br><span class="line"><span class="symbol">130 </span>[ <span class="number">0.29966027</span>  <span class="number">0.19986697</span>] [ <span class="number">0.40027133</span>]</span><br><span class="line"><span class="symbol">140 </span>[ <span class="number">0.29978839</span>  <span class="number">0.19991697</span>] [ <span class="number">0.40016907</span>]</span><br><span class="line"><span class="symbol">150 </span>[ <span class="number">0.29986817</span>  <span class="number">0.19994824</span>] [ <span class="number">0.40010536</span>]</span><br><span class="line"><span class="symbol">160 </span>[ <span class="number">0.29991791</span>  <span class="number">0.1999677</span> ] [ <span class="number">0.40006563</span>]</span><br><span class="line"><span class="symbol">170 </span>[ <span class="number">0.29994887</span>  <span class="number">0.19997987</span>] [ <span class="number">0.40004089</span>]</span><br><span class="line"><span class="symbol">180 </span>[ <span class="number">0.29996812</span>  <span class="number">0.19998746</span>] [ <span class="number">0.40002549</span>]</span><br><span class="line"><span class="symbol">190 </span>[ <span class="number">0.29998016</span>  <span class="number">0.19999218</span>] [ <span class="number">0.40001586</span>]</span><br><span class="line"><span class="symbol">200 </span>[ <span class="number">0.29998764</span>  <span class="number">0.19999513</span>] [ <span class="number">0.40000987</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，随着训练的进行，w 和 b 也慢慢接近真实的值，拟合越来越精确，接近正确的值。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>以上便是通过一个最简单的平面拟合的案例来说明了一下 TensorFlow 的用法，是不是很简单？</p>
                  <h2 id="代码"><a href="#代码" class="headerlink" title="代码"></a>代码</h2>
                  <p>本节代码地址：<a href="https://github.com/AIDeepLearning/TensorFlowBasis" target="_blank" rel="noopener">https://github.com/AIDeepLearning/TensorFlowBasis</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-12-09 00:24:47" itemprop="dateCreated datePublished" datetime="2017-12-09T00:24:47+08:00">2017-12-09</time>
                </span>
                <span id="/4893.html" class="post-meta-item leancloud_visitors" data-flag-title="TensorFlow基础入门" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>6.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>6 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4889.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Linux <i class="label-arrow"></i>
                  </a>
                  <a href="/4889.html" class="post-title-link" itemprop="url">小白学爬虫-批量部署Splash负载集群</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021225948.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021225948.jpg" alt=""></a> 部署公司生产环境的Splash集群无奈节点太多 差点被搞死·· 还好我有运维神器Ansible，一次编撰终生可用啊！而且这玩意儿 等幂特性 扩容回滚 So Easy！！ 闲话少说开搞！</p>
                  <h2 id="安装Ansible："><a href="#安装Ansible：" class="headerlink" title="安装Ansible："></a>安装Ansible：</h2>
                  <p>看官方文档去：<a href="http://www.ansible.com.cn/index.html" target="_blank" rel="noopener">http://www.ansible.com.cn/index.html</a> 好像这个主控端不支持Windows？ 大家虚拟机装个Ubuntu吧。</p>
                  <h2 id="闲话少扯直接上干货："><a href="#闲话少扯直接上干货：" class="headerlink" title="闲话少扯直接上干货："></a>闲话少扯直接上干货：</h2>
                  <p>整体目录如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">study@study:~/文档/ansible-examples$ tree Splash_Load_balancing_cluster</span><br><span class="line">Splash_Load_balancing_cluster</span><br><span class="line">├── group_vars</span><br><span class="line">│   └── all</span><br><span class="line">├── roles</span><br><span class="line">│   ├── common</span><br><span class="line">│   │   ├── files</span><br><span class="line">│   │   │   ├── CentOS-Base.repo</span><br><span class="line">│   │   │   ├── docker-ce.repo</span><br><span class="line">│   │   │   ├── epel.repo</span><br><span class="line">│   │   │   ├── ntp.conf</span><br><span class="line">│   │   │   └── RPM-GPG-KEY-EPEL-<span class="number">7</span></span><br><span class="line">│   │   ├── tasks</span><br><span class="line">│   │   │   └── main.yml</span><br><span class="line">│   │   └── templates</span><br><span class="line">│   ├── docker</span><br><span class="line">│   │   ├── handlers</span><br><span class="line">│   │   │   └── main.yml</span><br><span class="line">│   │   ├── tasks</span><br><span class="line">│   │   │   └── main.yml</span><br><span class="line">│   │   └── templates</span><br><span class="line">│   │       └── daemon<span class="selector-class">.json</span>.j2</span><br><span class="line">│   ├── haproxy</span><br><span class="line">│   │   ├── handlers</span><br><span class="line">│   │   │   └── main.yml</span><br><span class="line">│   │   ├── tasks</span><br><span class="line">│   │   │   └── main.yml</span><br><span class="line">│   │   └── templates</span><br><span class="line">│   │       └── haproxy<span class="selector-class">.cfg</span>.j2</span><br><span class="line">│   └── splash</span><br><span class="line">│       ├── files</span><br><span class="line">│       │   ├── filters</span><br><span class="line">│       │   │   └── default.txt</span><br><span class="line">│       │   ├── js-profiles</span><br><span class="line">│       │   ├── lua_modules</span><br><span class="line">│       │   └── proxy-profiles</span><br><span class="line">│       │       └── proxy.ini</span><br><span class="line">│       └── tasks</span><br><span class="line">│           └── main.yml</span><br><span class="line">├── site.retry</span><br><span class="line">└── site.yml</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> Group_vars： 里面定义全局使用的变量 Roles: 存放所有的规则目录 Roles/common :所有服务器初始化配置部署 Roles/common/filters :需要使用的文件或者文件夹 Roles/common/task：部署任务（main.yml为入口必须要有） Roles/common/templates :配置模板（jinja2模板语法 用于可变更的配置文件，可获取定义在Group_vars中的变量） Roles/Docker ：Docker的安装配置 Roles/HAproxy ： HAproxy的负载均衡配置 Roles/Splash : Splash的镜像拉取配置部署以及启动 site.yml ： 启动入口</p>
                  <h2 id="使用方法："><a href="#使用方法：" class="headerlink" title="使用方法："></a>使用方法：</h2>
                  <h4 id="在你的Inventory文件定义好主机分组："><a href="#在你的Inventory文件定义好主机分组：" class="headerlink" title="在你的Inventory文件定义好主机分组："></a>在你的Inventory文件定义好主机分组：</h4>
                  <p>必须包括HaProxy、和Docker两个分组如下：</p>
                  <figure class="highlight accesslog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">study@study:~/文档/ansible-examples$ cat /etc/ansible/inventory/splash </span><br><span class="line"><span class="string">[docker]</span></span><br><span class="line"><span class="number">1.1.1.1</span></span><br><span class="line"><span class="string">[haproxy]</span></span><br><span class="line"><span class="number">10.253.20.25</span></span><br><span class="line"></span><br><span class="line"><span class="string">[splash_ports]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="主控端新建SSH秘钥并发布到你你需要配置的所有主机！！！！（一定要注意如果本机当前工作用户在远程主机不存在额时候，需要指定remote-user这个参数）："><a href="#主控端新建SSH秘钥并发布到你你需要配置的所有主机！！！！（一定要注意如果本机当前工作用户在远程主机不存在额时候，需要指定remote-user这个参数）：" class="headerlink" title="主控端新建SSH秘钥并发布到你你需要配置的所有主机！！！！（一定要注意如果本机当前工作用户在远程主机不存在额时候，需要指定remote_user这个参数）："></a>主控端新建SSH秘钥并发布到你你需要配置的所有主机！！！！（一定要注意如果本机当前工作用户在远程主机不存在额时候，需要指定remote_user这个参数）：</h4>
                  <figure class="highlight elixir">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">study<span class="variable">@study</span><span class="symbol">:~/</span>文档/ansible-examples<span class="variable">$ </span>cat /etc/ansible/ansible.cfg </span><br><span class="line">[defaults]</span><br><span class="line">inventory= <span class="regexp">/etc/ansible</span><span class="regexp">/inventory/</span></span><br><span class="line"></span><br><span class="line">remote_user=root</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 好了开始执行：</p>
                  <figure class="highlight elixir">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">study<span class="variable">@study</span><span class="symbol">:~/</span>文档/ansible-examples/Splash_Load_balancing_cluster<span class="variable">$ </span>ansible-playbook site.yml</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 效果就像这样：</p>
                  <figure class="highlight markdown">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">PLAY [all] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="emphasis">***</span></span><br><span class="line"></span><br><span class="line">TASK [Gathering Facts] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span>*</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line"></span><br><span class="line">TASK [common : Copy the CentOS repository definition] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span></span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [common : Copy the Docker repository definition] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span></span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [common : Create the repository for EPEL] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span>**</span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [common : Create the GPG key for EPEL] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span></span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [common : Firewalld service stop] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span></span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [common : Chronyd service stop] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span>**</span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [common : Install Ansible Base package] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="emphasis">***</span>*</span><br><span class="line">ok: [10.1.4.100] =&gt; (item=['libselinux-python', 'libsemanage-python', 'ntp'])</span><br><span class="line">ok: [10.1.4.101] =&gt; (item=['libselinux-python', 'libsemanage-python', 'ntp'])</span><br><span class="line"></span><br><span class="line">TASK [common : Configure SELinux to disable] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="emphasis">***</span>*</span><br><span class="line"> [WARNING]: SELinux state change will take effect next reboot</span><br><span class="line"></span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [common : Change TimeZone] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span>**</span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [common : Copy NTP conf] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="emphasis">***</span>*</span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [common : NTP Start] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="emphasis">***</span></span><br><span class="line">ok: [10.1.4.100]</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">PLAY [docker] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span></span><br><span class="line"></span><br><span class="line">TASK [Gathering Facts] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span>*</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [docker : Install Docker package] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span></span><br><span class="line">ok: [10.1.4.101] =&gt; (item=['yum-utils', 'device-mapper-persistent-data', 'lvm2', 'docker-ce'])</span><br><span class="line"></span><br><span class="line">TASK [docker : Start Docker] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span></span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [docker : Create Docker Speed Configuration file] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="emphasis">***</span>*</span><br><span class="line">ok: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [docker : Restart Docker] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="emphasis">***</span></span><br><span class="line">changed: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [splash : pull splash] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span>*</span><br><span class="line">changed: [10.1.4.101]</span><br><span class="line"></span><br><span class="line">TASK [splash : Copy Splash module] <span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="strong">*****</span><span class="emphasis">***</span>*</span><br><span class="line">ok: [10.1.4.101] =&gt; (item=filters)</span><br><span class="line">ok: [10.1.4.101] =&gt; (item=js-profiles)</span><br><span class="line">ok: [10.1.4.101] =&gt; (item=lua_modules)</span><br><span class="line">ok: [10.1.4.101] =&gt; (item=proxy-profiles)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 静静等着跑完 就可以愉快的使用啦 ！ 需要增加节点的话直接把IP加载Docker分组下 重新执行一遍就可以了！ 需要注意如果SSH非默认的22端口还需要指定你的端口号！怎么指定 看看文档去 以上完毕！！！ 完整的看这儿：<a href="https://github.com/thsheep/ansible-examples" target="_blank" rel="noopener">https://github.com/thsheep/ansible-examples</a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-12-02 11:32:07" itemprop="dateCreated datePublished" datetime="2017-12-02T11:32:07+08:00">2017-12-02</time>
                </span>
                <span id="/4889.html" class="post-meta-item leancloud_visitors" data-flag-title="小白学爬虫-批量部署Splash负载集群" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>6.7k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>6 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4886.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4886.html" class="post-title-link" itemprop="url">小白学爬虫-在无GUI的CentOS上使用Selenium+Chrome</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>2019年01月04日16:32:17 更新了新的Chrome镜像 将Python版本升级到了3.7 Note: 推荐使用结尾提供的Docker镜像进行二次打包运行代码 各位小伙伴儿的采集日常是不是被JavaScript的各种点击事件折腾的欲仙欲死啊？好不容易找到个Selenium+Chrome可以解决问题！ 但是另一个▄█▀█●的事实摆在面前，服务器都特么没有GUI啊·· <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/20160124759183737.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/20160124759183737.gif" alt=""></a> 好吧！咱们要知难而上！决不能被这个点小困难打倒······· 然而摆在面前的事实是···· 他丫的各种装不上啊！坑爹啊！ 那么我来拯救你们于水火之间了！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg" alt=""></a> 服务器如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">[root@spider01</span> <span class="string">~]#</span> <span class="string">hostnamectl</span> </span><br><span class="line">   <span class="attr">Static hostname:</span> <span class="string">spider01</span></span><br><span class="line">         <span class="attr">Icon name:</span> <span class="string">computer-vm</span></span><br><span class="line">           <span class="attr">Chassis:</span> <span class="string">vm</span></span><br><span class="line">        <span class="attr">Machine ID:</span> <span class="string">1c4029c4e7fd42498e25bb75101f85b6</span></span><br><span class="line">           <span class="attr">Boot ID:</span> <span class="string">f5a67454b94b454fae3d75ef1ccab69f</span></span><br><span class="line">    <span class="attr">Virtualization:</span> <span class="string">kvm</span></span><br><span class="line">  <span class="attr">Operating System:</span> <span class="string">CentOS</span> <span class="string">Linux</span> <span class="number">7</span> <span class="string">(Core)</span></span><br><span class="line">       <span class="attr">CPE OS Name:</span> <span class="string">cpe:/o:centos:centos:7</span></span><br><span class="line">            <span class="attr">Kernel:</span> <span class="string">Linux</span> <span class="number">3.10</span><span class="number">.0</span><span class="number">-514.6</span><span class="number">.2</span><span class="string">.el7.x86_64</span></span><br><span class="line">      <span class="attr">Architecture:</span> <span class="string">x86-64</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 安装Chromeium:</p>
                  <figure class="highlight autoit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">## 安装yum源</span></span><br><span class="line">[root<span class="symbol">@spider01</span> ~]<span class="meta"># sudo yum install -y epel-release</span></span><br><span class="line"><span class="meta">## 安装Chrome</span></span><br><span class="line">[root<span class="symbol">@spider01</span> ~]<span class="meta"># yum install -y chromium</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 去这个地方：<a href="https://sites.google.com/a/chromium.org/chromedriver/downloads" target="_blank" rel="noopener">https://sites.google.com/a/chromium.org/chromedriver/downloads</a> 下载ChromeDriver驱动放在/usr/bin/目录下： 完成结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">root@</span>spider01 ~]# ll /usr/bin/ | grep chrom</span><br><span class="line">-rwxrwxrwx. <span class="number">1</span> root root   <span class="number">7500280</span> <span class="number">11</span>月 <span class="number">29</span> <span class="number">17</span>:<span class="number">32</span> chromedriver</span><br><span class="line">lrwxrwxrwx. <span class="number">1</span> root root        <span class="number">47</span> <span class="number">11</span>月 <span class="number">30</span> <span class="number">09</span>:<span class="number">35</span> chromium-browser -&gt; /usr/lib64/chromium-browser/chromium-browser.sh</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 安装XVFB：</p>
                  <figure class="highlight autoit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[root<span class="symbol">@spider01</span> ~]<span class="meta"># yum install Xvfb -y</span></span><br><span class="line">[root<span class="symbol">@spider01</span> ~]<span class="meta"># yum install xorg-x11-fonts* -y</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 新建在/usr/bin/ 一个名叫 xvfb-chromium 的文件写入以下内容：</p>
                  <figure class="highlight bash">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[root@spider01 ~]<span class="comment"># cat /usr/bin/xvfb-chromium </span></span><br><span class="line"><span class="meta">#!/bin/bash</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="title">_kill_procs</span></span>() &#123;</span><br><span class="line">  <span class="built_in">kill</span> -TERM <span class="variable">$chromium</span></span><br><span class="line">  <span class="built_in">wait</span> <span class="variable">$chromium</span></span><br><span class="line">  <span class="built_in">kill</span> -TERM <span class="variable">$xvfb</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># Setup a trap to catch SIGTERM and relay it to child processes</span></span><br><span class="line"><span class="built_in">trap</span> _kill_procs SIGTERM</span><br><span class="line"></span><br><span class="line">XVFB_WHD=<span class="variable">$&#123;XVFB_WHD:-1280x720x16&#125;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># Start Xvfb</span></span><br><span class="line">Xvfb :99 -ac -screen 0 <span class="variable">$XVFB_WHD</span> -nolisten tcp &amp;</span><br><span class="line">xvfb=$!</span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> DISPLAY=:99</span><br><span class="line"></span><br><span class="line">chromium --no-sandbox --<span class="built_in">disable</span>-gpu<span class="variable">$@</span> &amp;</span><br><span class="line">chromium=$!</span><br><span class="line"></span><br><span class="line"><span class="built_in">wait</span> <span class="variable">$chromium</span></span><br><span class="line"><span class="built_in">wait</span> <span class="variable">$xvfb</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 更改软连接：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">## 更改Chrome启动的软连接</span><br><span class="line">[<span class="symbol">root@</span>spider01 ~]# ln -s /usr/lib64/chromium-browser/chromium-browser.sh /usr/bin/chromium</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[<span class="symbol">root@</span>spider01 ~]# rm -rf /usr/bin/chromium-browser</span><br><span class="line"></span><br><span class="line">[<span class="symbol">root@</span>spider01 ~]# ln -s /usr/bin/xvfb-chromium /usr/bin/chromium-browser</span><br><span class="line"></span><br><span class="line">[<span class="symbol">root@</span>spider01 ~]# ln -s /usr/bin/xvfb-chromium /usr/bin/google-chrome</span><br><span class="line"></span><br><span class="line">[<span class="symbol">root@</span>spider01 ~]# ll /usr/bin/ | grep chrom*</span><br><span class="line">-rwxrwxrwx. <span class="number">1</span> root root   <span class="number">7500280</span> <span class="number">11</span>月 <span class="number">29</span> <span class="number">17</span>:<span class="number">32</span> chromedriver</span><br><span class="line">lrwxrwxrwx. <span class="number">1</span> root root        <span class="number">47</span> <span class="number">11</span>月 <span class="number">30</span> <span class="number">09</span>:<span class="number">47</span> chromium -&gt; /usr/lib64/chromium-browser/chromium-browser.sh</span><br><span class="line">lrwxrwxrwx. <span class="number">1</span> root root        <span class="number">22</span> <span class="number">11</span>月 <span class="number">30</span> <span class="number">09</span>:<span class="number">48</span> chromium-browser -&gt; /usr/bin/xvfb-chromium</span><br><span class="line">-rwxr-xr-x. <span class="number">1</span> root root     <span class="number">73848</span> <span class="number">12</span>月  <span class="number">7</span> <span class="number">2016</span> chronyc</span><br><span class="line">lrwxrwxrwx. <span class="number">1</span> root root        <span class="number">22</span> <span class="number">11</span>月 <span class="number">30</span> <span class="number">09</span>:<span class="number">48</span> google-chrome -&gt; /usr/bin/xvfb-chromium</span><br><span class="line">-rwxrwxrwx. <span class="number">1</span> root root       <span class="number">387</span> <span class="number">11</span>月 <span class="number">29</span> <span class="number">18</span>:<span class="number">16</span> xvfb-chromium</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 来瞅瞅能不能用哦：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">&gt;&gt;</span>&gt; from selenium import webdriver</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; options = webdriver.ChromeOptions()</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; options.add_argument(<span class="string">'--headless'</span>)</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; options.add_argument(<span class="string">'--no-sandbox'</span>)</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; options.add_argument(<span class="string">'--disable-extensions'</span>)</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; options.add_argument(<span class="string">'--disable-gpu'</span>)</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; driver = webdriver.Chrome(options=options)</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; driver.get(<span class="string">"http://www.baidu.com"</span>)</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; driver.find_element_by_xpath(<span class="string">"./*//input[@id='kw']"</span>).send_keys(<span class="string">"哎哟卧槽"</span>)</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; driver.find_element_by_xpath(<span class="string">"./*//input[@id='su']"</span>).click()</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; driver.page_source</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021223818.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021223818.jpg" alt=""></a><strong>No problem！！！！</strong> 好了部署完了！当然Docker这么火贼适合懒人了！来来 看这儿 Docker版的 妥妥滴！</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">docker pull thsheep/python:<span class="number">3.7</span>-debian-chrome</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>做好了Python3.7和Chrome集成 需要自己使用Dockerfile来重新打包安装你需要的Python包。</p>
                  <h3 id="Note-请按照以下方式初始化Webdriver！！！！！！！！"><a href="#Note-请按照以下方式初始化Webdriver！！！！！！！！" class="headerlink" title="Note: 请按照以下方式初始化Webdriver！！！！！！！！"></a><strong>Note: 请按照以下方式初始化Webdriver！！！！！！！！</strong></h3>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from selenium import webdriver</span><br><span class="line"></span><br><span class="line">options = webdriver.<span class="constructor">ChromeOptions()</span></span><br><span class="line">options.add<span class="constructor">_argument('--<span class="params">headless</span>')</span></span><br><span class="line">options.add<span class="constructor">_argument('--<span class="params">no</span>-<span class="params">sandbox</span>')</span></span><br><span class="line">options.add<span class="constructor">_argument('--<span class="params">disable</span>-<span class="params">extensions</span>')</span></span><br><span class="line">options.add<span class="constructor">_argument('--<span class="params">disable</span>-<span class="params">gpu</span>')</span></span><br><span class="line"></span><br><span class="line">driver = webdriver.<span class="constructor">Chrome(<span class="params">options</span>=<span class="params">options</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="否则会出现无法初始化Webdriver的情况"><a href="#否则会出现无法初始化Webdriver的情况" class="headerlink" title="否则会出现无法初始化Webdriver的情况"></a>否则会出现无法初始化Webdriver的情况</h2>
                  <h2 id="顺便一提！！！！这个玩意儿从事Web测试工作的小伙伴可以用！！！！！！！！"><a href="#顺便一提！！！！这个玩意儿从事Web测试工作的小伙伴可以用！！！！！！！！" class="headerlink" title="顺便一提！！！！这个玩意儿从事Web测试工作的小伙伴可以用！！！！！！！！"></a>顺便一提！！！！这个玩意儿从事Web测试工作的小伙伴可以用！！！！！！！！</h2>
                  <p> 下面是Dockerfile文件：</p>
                  <figure class="highlight jboss-cli">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">FROM python<span class="function">:3.7-stretch</span></span><br><span class="line"></span><br><span class="line">ENV DBUS_SESSION_BUS_ADDRESS=<span class="string">/dev/null</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#============================================</span></span><br><span class="line"><span class="comment"># Google Chrome</span></span><br><span class="line"><span class="comment">#============================================</span></span><br><span class="line">RUN wget -q -O - https:<span class="string">//dl-ssl.google.com/linux/linux_signing_key.pub</span> | apt-key add - &amp;&amp; \</span><br><span class="line"><span class="keyword">echo</span> <span class="string">"deb http://dl.google.com/linux/chrome/deb/ stable main"</span> &gt;&gt; <span class="string">/etc/apt/sources.list.d/google-chrome.list</span> &amp;&amp; \</span><br><span class="line">apt-get update -qqy &amp;&amp; \</span><br><span class="line">apt-get -qqy install google-chrome-stable unzip&amp;&amp; \</span><br><span class="line">rm <span class="string">/etc/apt/sources.list.d/google-chrome.list</span> &amp;&amp; \</span><br><span class="line">rm -rf <span class="string">/var/lib/apt/lists/</span>* <span class="string">/var/cache/apt/</span>*</span><br><span class="line"></span><br><span class="line"><span class="comment">#==================</span></span><br><span class="line"><span class="comment"># Chrome driver</span></span><br><span class="line"><span class="comment"># CHROME_DRIVER_VERSION 需要根据上面安装的Chrome版本来设置（最好设置成最新版本）</span></span><br><span class="line"><span class="comment"># http://chromedriver.chromium.org/downloads 版本号在这页面上查看</span></span><br><span class="line"><span class="comment">#==================</span></span><br><span class="line">ARG CHROME_DRIVER_VERSION=2.45</span><br><span class="line">RUN wget -O <span class="string">/tmp/chromedriver.zip</span> https:<span class="string">//chromedriver.storage.googleapis.com/</span>$CHROME_DRIVER_VERSION/chromedriver_linux64.zip &amp;&amp; \</span><br><span class="line">rm -rf <span class="string">/opt/selenium/chromedriver</span> &amp;&amp; \</span><br><span class="line">unzip <span class="string">/tmp/chromedriver.zip</span> -d <span class="string">/opt/selenium</span> &amp;&amp; \</span><br><span class="line">rm <span class="string">/tmp/chromedriver.zip</span> &amp;&amp; \</span><br><span class="line">mv <span class="string">/opt/selenium/chromedriver</span> <span class="string">/opt/selenium/chromedriver-</span>$CHROME_DRIVER_VERSION &amp;&amp; \</span><br><span class="line">chmod 755 <span class="string">/opt/selenium/chromedriver-</span>$CHROME_DRIVER_VERSION &amp;&amp; \</span><br><span class="line">ln -fs <span class="string">/opt/selenium/chromedriver-</span>$CHROME_DRIVER_VERSION <span class="string">/usr/bin/chromedriver</span></span><br><span class="line"></span><br><span class="line">RUN google-chrome-stable <span class="params">--version</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-11-30 10:32:12" itemprop="dateCreated datePublished" datetime="2017-11-30T10:32:12+08:00">2017-11-30</time>
                </span>
                <span id="/4886.html" class="post-meta-item leancloud_visitors" data-flag-title="小白学爬虫-在无GUI的CentOS上使用Selenium+Chrome" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>4.7k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4880.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4880.html" class="post-title-link" itemprop="url">小白学爬虫-设置Selenium+Chrome代理</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/20160124759183737.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/20160124759183737.gif" alt=""></a> 微博登录限制了错误次数···加上Cookie大批账号被封需要从Cookie池中 剔除被封的账号··· 需要使用代理··· 无赖百度了大半天都是特么的啥玩意儿？？？结果换成了 Google手到擒来 分分钟解决（那么问题来了？百度除了卖假药还会干啥？） <strong>Selenium+Chrome认证代理不能通过options处理。只能换个方法使用扩展解决</strong> 原文地址：<a href="https://stackoverflow.com/questions/29983106/how-can-i-set-proxy-with-authentication-in-selenium-chrome-web-driver-using-pyth#answer-30953780" target="_blank" rel="noopener">https://stackoverflow.com/questions/29983106/how-can-i-set-proxy-with-authentication-in-selenium-chrome-web-driver-using-pyth#answer-30953780</a> （Stack Overflow 这是个好地方啊） <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg" alt=""></a> 走你！</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># @Time    : 2017/11/15 9:50</span></span><br><span class="line"><span class="comment"># @Author  : 哎哟卧槽</span></span><br><span class="line"><span class="comment"># @Site    : </span></span><br><span class="line"><span class="comment"># @File    : pubilc.py</span></span><br><span class="line"><span class="comment"># @Software: PyCharm</span></span><br><span class="line"></span><br><span class="line">import string</span><br><span class="line">import zipfile</span><br><span class="line"></span><br><span class="line">def create_proxyauth_extension(proxy_host, proxy_port,</span><br><span class="line">                               proxy_username, proxy_password,</span><br><span class="line">                               <span class="attribute">scheme</span>=<span class="string">'http'</span>, <span class="attribute">plugin_path</span>=None):</span><br><span class="line">    <span class="string">""</span><span class="string">"代理认证插件</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    args:</span></span><br><span class="line"><span class="string">        proxy_host (str): 你的代理地址或者域名（str类型）</span></span><br><span class="line"><span class="string">        proxy_port (int): 代理端口号（int类型）</span></span><br><span class="line"><span class="string">        proxy_username (str):用户名（字符串）</span></span><br><span class="line"><span class="string">        proxy_password (str): 密码 （字符串）</span></span><br><span class="line"><span class="string">    kwargs:</span></span><br><span class="line"><span class="string">        scheme (str): 代理方式 默认http</span></span><br><span class="line"><span class="string">        plugin_path (str): 扩展的绝对路径</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    return str -&gt; plugin_path</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    </span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> plugin_path is None:</span><br><span class="line">        plugin_path = <span class="string">'vimm_chrome_proxyauth_plugin.zip'</span></span><br><span class="line"></span><br><span class="line">    manifest_json = <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">    &#123;</span></span><br><span class="line"><span class="string">        "</span>version<span class="string">": "</span>1.0.0<span class="string">",</span></span><br><span class="line"><span class="string">        "</span>manifest_version<span class="string">": 2,</span></span><br><span class="line"><span class="string">        "</span>name<span class="string">": "</span>Chrome Proxy<span class="string">",</span></span><br><span class="line"><span class="string">        "</span>permissions<span class="string">": [</span></span><br><span class="line"><span class="string">            "</span>proxy<span class="string">",</span></span><br><span class="line"><span class="string">            "</span>tabs<span class="string">",</span></span><br><span class="line"><span class="string">            "</span>unlimitedStorage<span class="string">",</span></span><br><span class="line"><span class="string">            "</span>storage<span class="string">",</span></span><br><span class="line"><span class="string">            "</span>&lt;all_urls&gt;<span class="string">",</span></span><br><span class="line"><span class="string">            "</span>webRequest<span class="string">",</span></span><br><span class="line"><span class="string">            "</span>webRequestBlocking<span class="string">"</span></span><br><span class="line"><span class="string">        ],</span></span><br><span class="line"><span class="string">        "</span>background<span class="string">": &#123;</span></span><br><span class="line"><span class="string">            "</span>scripts<span class="string">": ["</span>background.js<span class="string">"]</span></span><br><span class="line"><span class="string">        &#125;,</span></span><br><span class="line"><span class="string">        "</span>minimum_chrome_version<span class="string">":"</span>22.0.0<span class="string">"</span></span><br><span class="line"><span class="string">    &#125;</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line"></span><br><span class="line">    background_js = string.Template(</span><br><span class="line">    <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">    var config = &#123;</span></span><br><span class="line"><span class="string">            mode: "</span>fixed_servers<span class="string">",</span></span><br><span class="line"><span class="string">            rules: &#123;</span></span><br><span class="line"><span class="string">              singleProxy: &#123;</span></span><br><span class="line"><span class="string">                scheme: "</span><span class="variable">$&#123;scheme&#125;</span><span class="string">",</span></span><br><span class="line"><span class="string">                host: "</span><span class="variable">$&#123;host&#125;</span><span class="string">",</span></span><br><span class="line"><span class="string">                port: parseInt(<span class="variable">$&#123;port&#125;</span>)</span></span><br><span class="line"><span class="string">              &#125;,</span></span><br><span class="line"><span class="string">              bypassList: ["</span>foobar.com<span class="string">"]</span></span><br><span class="line"><span class="string">            &#125;</span></span><br><span class="line"><span class="string">          &#125;;</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    chrome.proxy.settings.set(&#123;value: config, scope: "</span>regular<span class="string">"&#125;, function() &#123;&#125;);</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    function callbackFn(details) &#123;</span></span><br><span class="line"><span class="string">        return &#123;</span></span><br><span class="line"><span class="string">            authCredentials: &#123;</span></span><br><span class="line"><span class="string">                username: "</span><span class="variable">$&#123;username&#125;</span><span class="string">",</span></span><br><span class="line"><span class="string">                password: "</span><span class="variable">$&#123;password&#125;</span><span class="string">"</span></span><br><span class="line"><span class="string">            &#125;</span></span><br><span class="line"><span class="string">        &#125;;</span></span><br><span class="line"><span class="string">    &#125;</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    chrome.webRequest.onAuthRequired.addListener(</span></span><br><span class="line"><span class="string">                callbackFn,</span></span><br><span class="line"><span class="string">                &#123;urls: ["</span>&lt;all_urls&gt;<span class="string">"]&#125;,</span></span><br><span class="line"><span class="string">                ['blocking']</span></span><br><span class="line"><span class="string">    );</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    ).substitute(</span><br><span class="line">        <span class="attribute">host</span>=proxy_host,</span><br><span class="line">        <span class="attribute">port</span>=proxy_port,</span><br><span class="line">        <span class="attribute">username</span>=proxy_username,</span><br><span class="line">        <span class="attribute">password</span>=proxy_password,</span><br><span class="line">        <span class="attribute">scheme</span>=scheme,</span><br><span class="line">    )</span><br><span class="line">    with zipfile.ZipFile(plugin_path, <span class="string">'w'</span>) as zp:</span><br><span class="line">        zp.writestr(<span class="string">"manifest.json"</span>, manifest_json)</span><br><span class="line">        zp.writestr(<span class="string">"background.js"</span>, background_js)</span><br><span class="line"></span><br><span class="line">    return plugin_path</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>使用方法：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line"><span class="keyword">from</span> common.pubilc import create_proxyauth_extension</span><br><span class="line"></span><br><span class="line">proxyauth_plugin_path = create_proxyauth_extension(</span><br><span class="line">    <span class="attribute">proxy_host</span>=<span class="string">"XXXXX.com"</span>,</span><br><span class="line">    <span class="attribute">proxy_port</span>=9020,</span><br><span class="line">    <span class="attribute">proxy_username</span>=<span class="string">"XXXXXXX"</span>,</span><br><span class="line">    <span class="attribute">proxy_password</span>=<span class="string">"XXXXXXX"</span></span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">co = webdriver.ChromeOptions()</span><br><span class="line"><span class="comment"># co.add_argument("--start-maximized")</span></span><br><span class="line">co.add_extension(proxyauth_plugin_path)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">driver = webdriver.Chrome(<span class="attribute">executable_path</span>=<span class="string">"C:\chromedriver.exe"</span>, <span class="attribute">chrome_options</span>=co)</span><br><span class="line">driver.<span class="builtin-name">get</span>(<span class="string">"http://ip138.com/"</span>)</span><br><span class="line"><span class="builtin-name">print</span>(driver.page_source)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 无认证代理：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">options = webdriver.<span class="constructor">ChromeOptions()</span></span><br><span class="line">options.add<span class="constructor">_argument('--<span class="params">proxy</span>-<span class="params">server</span>=<span class="params">http</span>:<span class="operator">/</span><span class="operator">/</span><span class="params">ip</span>:<span class="params">port</span>')</span>  </span><br><span class="line">driver = webdriver.<span class="constructor">Chrome(<span class="params">executable_path</span>=<span class="string">"C:\chromedriver.exe"</span>, <span class="params">chrome_options</span>=0ptions)</span></span><br><span class="line">driver.get(<span class="string">"http://ip138.com/"</span>)</span><br><span class="line">print(driver.page_source)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <pre><code>以上完毕 So Easy
</code></pre>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-11-18 11:13:54" itemprop="dateCreated datePublished" datetime="2017-11-18T11:13:54+08:00">2017-11-18</time>
                </span>
                <span id="/4880.html" class="post-meta-item leancloud_visitors" data-flag-title="小白学爬虫-设置Selenium+Chrome代理" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4853.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4853.html" class="post-title-link" itemprop="url">一个采集系统的构建</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>整个系统: <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/10/WX20171017-225541.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/10/WX20171017-225541.png" alt=""></a> 采集系统： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/10/WX20171017-225831.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/10/WX20171017-225831.png" alt=""></a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-10-17 23:00:23" itemprop="dateCreated datePublished" datetime="2017-10-17T23:00:23+08:00">2017-10-17</time>
                </span>
                <span id="/4853.html" class="post-meta-item leancloud_visitors" data-flag-title="一个采集系统的构建" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>10</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4833.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Java <i class="label-arrow"></i>
                  </a>
                  <a href="/4833.html" class="post-title-link" itemprop="url">java基础之数据类型</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p><strong><strong>PS：此文章为小白提供，大佬请绕道！！！！</strong></strong> <strong>首先特别感谢大才哥给我提供这个平台，未来我希望把java这个版块的内容补全。</strong> 今天要讲的是数据类型，最最最基础的内容~ java标识符、数据类型、关键字 开始我们先看下如何注释java代码。 标识符：类名，方法名，变量。 有三种方式分别为 //表示注释一行代码 /<em> 表示注释一行或者多行代码 (从上面到下面都是注释的代码） </em>/ 下面还有一种注释方式叫做文档注释。 /<strong> 通常这样表示 */ 文档注释一般写在代码开头用来简述你所做程序的具体内容，在这之前我们首先看一下javadoc命令，我先编写一个简答的代码： package com.briup.chap02; /</strong> @author Twinkle @version 1.0 It’s a text file <em>/ public class PrimitiveType{ public static void main(String[] args){ byte b = 123; byte b1 = 300; } } 我们javadoc -d 生成目录 编译文件 编译成功后，我们打开刚刚生成doc里打开index.html看一下，大概是这样的： 类概要 类： Student 说明： It’s a text file 这样我们就可以看出文档注释的意义了，他可以显示在你编译出来文档的说明里，但有人会发现为啥我们编写出来的author没有出来呀？ 因为他的最前面有一个@，我们需要编写的时候把它加上去才能显示出来，现在我们来试一下， —javadoc -d bin/doc-author -version src/PrimitiveType.java，这样作者和版本信息就出来了 一.类名 这边我们要记住一些代码的基本格式： 类名的写法：Student（前面首字母要大写） 方法和变量的写法：genderItem（前面单词小写，后面单词开头要大写） 常量写法：MAX_PAGE（常量大写，中间一般加下划线） 二.关键字 关键字其实就是电脑里面已经定义好的有特殊意义的标识符，像int,for，double什么的都是关键字。具体意思请百度一下～ 三.数据类型 数据类型是这篇文章的重点，我们来看下这些基本的数据类型 类型 二进制位 例 范围 byte 8位 11111111～01111111 -2^7~2^7-1 short 16位 16个二进制代码 -2^15~2^15-1 int 32位 32个二进制代码 -2^31~2^31-1 long 64位 64个二进制代码 -2^63~2^63-1 浮点型: float 32位 32个二进制代码 double 64位 64个二进制代码 布尔型： boolean 只有false和true两种类型。 具体解释一下为什么会有这么多类型呢？而且二进制位为什么还不一样？ 类型多的原因是因为有些数值本身就很小，传递给大的数据类型的话，虽然可以进去，但是有些二进制位就空闲了，占用了多余的内存却没有什么作用，所以才会有这么多的类型。 我们知道编程最终的目的是我们把代码传递给硬件，通过硬件来工作，但是呢，硬件只识别二进制代码，所以java会有一个把它的代码转化为二进制代码的过渡，上面的二进制位就是二进制码的数目，我们要想看他的范围有多大，可以这样算，二进制的第一位为标志符，通俗一点讲就是正负号，后面还有n位的话它的范围就是-|2^n|～|2^n-1| 如果我们定义的类型超出这个范围的话(也就是盆子里已经装满了东西如果再加），java就会报错，超出指定的范围，所以当我们定义数据类型的时候要搞清楚各数据类型的范围。 还有一个特殊的数据类型：char (‘字符’) char的具体位数要结合unicode编码。问题又来了,unicode编码又是什么鬼！unicode编码是一个字符集，里面包含了中，日，韩，三种文字，我们可以通过char的方法来打印出字符:char(‘u\unicode编码’)，unicode表具体百度一下哈～ 数据类型转换： 显式转换：也就是强制转换 隐式转换：由JVM虚拟机自行转换 数据类型的强制转换：int a = (强制转换类型)b 转换规则:从存储范围大的类型到存储范围小的类型。 具体规则为：double→float→long→int→short(char)→byte byte b =10; byte a = (int) b; 如果我们把int类型的b转换给byte类型的a的话，会出现溢出现象，所以会报错。 所以正确强制转换的方式为～～： byte b = 10; int（或者更大的类型） a =(int) b; java基本的数据类型就讲到这里啦~ <em>*--可能发布的内容有点混乱，我会尽快把前面的补齐~有疑问的话可以到大才哥的群里找我哈~</em></em></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/Twinkle" class="author" itemprop="url" rel="index">Twinkle</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-09-27 18:17:06" itemprop="dateCreated datePublished" datetime="2017-09-27T18:17:06+08:00">2017-09-27</time>
                </span>
                <span id="/4833.html" class="post-meta-item leancloud_visitors" data-flag-title="java基础之数据类型" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4826.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> 未分类 <i class="label-arrow"></i>
                  </a>
                  <a href="/4826.html" class="post-title-link" itemprop="url">小白进阶第七篇（Splash负载均衡）</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>对于Scrapy处理Ajax 处理方式当然是同家兄弟Splash比较靠谱！ 但是Splash有个很坑爹的毛病就是负载承受相对较小·· 一不留神就GG了·········· 然后也就没有然后了~~！ 所以准备给Splash做一个负载均衡；后端放一大堆的Splash这样总不会GG了吧。 就算其中一个GG了还有其它的可替代不是？ 废话不多少开整·· 环境是基于： CentOS 7.3 Docker 17.06.2-ce Splash 3.0 HAproxy 1.7.9 （CentOS大家可以将yum切换为阿里云的yum源 Docker同理）</p>
                  <h4 id="阿里yum源：-http-mirrors-aliyun-com-help-centos-照葫芦画瓢做一遍（你是CentOS7啊！！！！不要选成其他版本了）"><a href="#阿里yum源：-http-mirrors-aliyun-com-help-centos-照葫芦画瓢做一遍（你是CentOS7啊！！！！不要选成其他版本了）" class="headerlink" title="阿里yum源： http://mirrors.aliyun.com/help/centos  照葫芦画瓢做一遍（你是CentOS7啊！！！！不要选成其他版本了）"></a>阿里yum源： <a href="http://mirrors.aliyun.com/help/centos" target="_blank" rel="noopener">http://mirrors.aliyun.com/help/centos</a> 照葫芦画瓢做一遍（你是CentOS7啊！！！！不要选成其他版本了）</h4>
                  <h2 id="注意以下只需要在你需要运行splash的机器上安装即可"><a href="#注意以下只需要在你需要运行splash的机器上安装即可" class="headerlink" title="注意以下只需要在你需要运行splash的机器上安装即可"></a><strong>注意以下只需要在你需要运行splash的机器上安装即可</strong></h2>
                  <h4 id="阿里Docker源："><a href="#阿里Docker源：" class="headerlink" title="阿里Docker源："></a>阿里Docker源：</h4>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># step 1: 安装必要的一些系统工具</span></span><br><span class="line"></span><br><span class="line">sudo yum <span class="keyword">install</span> -y yum-utils device-mapper-persistent-<span class="keyword">data</span> lvm2</span><br><span class="line"></span><br><span class="line"><span class="comment"># Step 2: 添加软件源信息</span></span><br><span class="line"></span><br><span class="line">sudo yum-config-manager <span class="comment">--add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># Step 3: 更新并安装 Docker-CE</span></span><br><span class="line"></span><br><span class="line">sudo yum makecache <span class="keyword">fast</span></span><br><span class="line">sudo yum -y <span class="keyword">install</span> docker-ce</span><br><span class="line"></span><br><span class="line"><span class="comment"># Step 4: 开启Docker服务</span></span><br><span class="line"></span><br><span class="line">sudo service docker <span class="keyword">start</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="安装Docker加速器："><a href="#安装Docker加速器：" class="headerlink" title="安装Docker加速器："></a>安装Docker加速器：</h4>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl -sSL http<span class="variable">s:</span>//<span class="built_in">get</span>.daocloud.io/daotools/set_mirror.<span class="keyword">sh</span> | <span class="keyword">sh</span> -s http://<span class="number">8050</span>f360.<span class="keyword">m</span>.daocloud.io</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="重启Docker："><a href="#重启Docker：" class="headerlink" title="重启Docker："></a>重启Docker：</h4>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">systemctl restart docker</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样可以极快的速度拉取镜像。 获取splash最新的docker镜像：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">docker pull scrapinghub/splash:<span class="literal">master</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="关闭所有机器防火墙firewalld-网络安全的环境关闭，不安全的环境请放行端口，自行百度"><a href="#关闭所有机器防火墙firewalld-网络安全的环境关闭，不安全的环境请放行端口，自行百度" class="headerlink" title="关闭所有机器防火墙firewalld(网络安全的环境关闭，不安全的环境请放行端口，自行百度):"></a>关闭所有机器防火墙firewalld(网络安全的环境关闭，不安全的环境请放行端口，自行百度):</h4>
                  <figure class="highlight gauss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">systemctl <span class="keyword">disable</span> firewalld</span><br><span class="line"></span><br><span class="line">systemctl <span class="keyword">stop</span> firewalld</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="创建Splash配置文件目录："><a href="#创建Splash配置文件目录：" class="headerlink" title="创建Splash配置文件目录："></a>创建Splash配置文件目录：</h4>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># 存放过滤规则文件的目录</span><br><span class="line"></span><br><span class="line">[<span class="symbol">root@</span>localhost ~]# mkdir filters</span><br><span class="line"></span><br><span class="line"># 存放JavaScript文件目录</span><br><span class="line"></span><br><span class="line">[<span class="symbol">root@</span>localhost ~]# mkdir js-profiles</span><br><span class="line"></span><br><span class="line"># 存放lua模块的目录</span><br><span class="line"></span><br><span class="line">[<span class="symbol">root@</span>localhost ~]# mkdir lua_modules</span><br><span class="line"></span><br><span class="line"># 存放代理文件的目录</span><br><span class="line"></span><br><span class="line">[<span class="symbol">root@</span>localhost ~]# mkdir proxy-profiles</span><br><span class="line"></span><br><span class="line"># 创建完成如下：</span><br><span class="line"></span><br><span class="line">[<span class="symbol">root@</span>localhost ~]# pwd</span><br><span class="line">/root</span><br><span class="line">[<span class="symbol">root@</span>localhost ~]# ll</span><br><span class="line">total <span class="number">4</span></span><br><span class="line">drwxr-xr-x. <span class="number">2</span> root root   <span class="number">25</span> Sep <span class="number">26</span> <span class="number">03</span>:<span class="number">00</span> filters</span><br><span class="line">drwxr-xr-x. <span class="number">2</span> root root    <span class="number">6</span> Sep <span class="number">25</span> <span class="number">21</span>:<span class="number">08</span> js-profiles</span><br><span class="line">drwxr-xr-x. <span class="number">2</span> root root    <span class="number">6</span> Sep <span class="number">25</span> <span class="number">21</span>:<span class="number">08</span> lua_modules</span><br><span class="line">drwxr-xr-x. <span class="number">2</span> root root   <span class="number">32</span> Sep <span class="number">25</span> <span class="number">21</span>:<span class="number">08</span> proxy-profiles</span><br><span class="line">[<span class="symbol">root@</span>localhost ~]#</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="启动Splash："><a href="#启动Splash：" class="headerlink" title="启动Splash："></a>启动Splash：</h4>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">docker run -d -p <span class="number">8050</span>:<span class="number">8050</span> --memory=<span class="number">5.0</span>G --restart=always  --name splash       -v <span class="regexp">/root/</span>proxy-<span class="string">profiles:</span><span class="regexp">/etc/</span>splash<span class="regexp">/proxy-profiles       -v /</span>root<span class="regexp">/js-profiles:/</span>etc<span class="regexp">/splash/</span>js-profiles       -v <span class="regexp">/root/</span><span class="string">lua_modules:</span><span class="regexp">/etc/</span>splash<span class="regexp">/lua_modules       -v /</span>root<span class="regexp">/filters:/</span>etc<span class="regexp">/splash/</span>filters       scrapinghub/<span class="string">splash:</span>master --maxrss <span class="number">4500</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>docker run 启动一个容器 -d 后台启动 -p 8050:8050 将容器的8050端口和物理机的8050端口绑定（可以从8050端口访问容器服务应用） —memory=5.0G 容器最大使用内存为5.0GB，超出这个限制会被主进程杀死（使用free -mg 查看并酌情设置你的内存使用） —restart=always 容器退出后无条件重启（满了5GB被杀死，然后重启 释放内存） —name splash 容器的名字叫splash（可以忽略） -v <strong>**</strong> 三个-v参数是将宿主机的目录挂载进容器，便于容器能够直接访问挂载目录中的内容 scrapinghub/splash:master 用于启动容器的镜像 —maxrss 4500 Splash最大内存使用为4500MB</p>
                  <h4 id="查看容器是否启动："><a href="#查看容器是否启动：" class="headerlink" title="查看容器是否启动："></a>查看容器是否启动：</h4>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">root@</span>localhost ~]# docker ps -a</span><br><span class="line">CONTAINER ID        IMAGE                       COMMAND                  CREATED             STATUS              PORTS                              NAMES</span><br><span class="line"><span class="number">1</span>b34f7933095        scrapinghub/splash:master   <span class="string">"python3 /app/bin/..."</span>   <span class="number">4</span> hours ago         Up <span class="number">4</span> hours          <span class="number">5023</span>/tcp, <span class="number">0.0</span><span class="number">.0</span><span class="number">.0</span>:<span class="number">8050</span>-&gt;<span class="number">8050</span>/tcp   splash</span><br><span class="line">[<span class="symbol">root@</span>localhost ~]#</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="访问Splash是否正常工作："><a href="#访问Splash是否正常工作：" class="headerlink" title="访问Splash是否正常工作："></a>访问Splash是否正常工作：</h4>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/09/测试.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/09/测试.png" alt=""></a></p>
                  <h2 id="请注意：以上操作只需要在你需要运行splash的机器上安装即可"><a href="#请注意：以上操作只需要在你需要运行splash的机器上安装即可" class="headerlink" title="请注意：以上操作只需要在你需要运行splash的机器上安装即可"></a><strong>请注意：以上操作只需要在你需要运行splash的机器上安装即可</strong></h2>
                  <h1 id="安装HAproxy实现负载均衡："><a href="#安装HAproxy实现负载均衡：" class="headerlink" title="安装HAproxy实现负载均衡："></a>安装HAproxy实现负载均衡：</h1>
                  <h4 id="安装zlib-devel（HAproxy使用gzip功能）："><a href="#安装zlib-devel（HAproxy使用gzip功能）：" class="headerlink" title="安装zlib-devel（HAproxy使用gzip功能）："></a>安装zlib-devel（HAproxy使用gzip功能）：</h4>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">yum <span class="keyword">install</span> zlib-devel -y</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="安装HAproxy："><a href="#安装HAproxy：" class="headerlink" title="安装HAproxy："></a>安装HAproxy：</h4>
                  <figure class="highlight autoit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta"># 个人喜好 源码放在这个目录</span></span><br><span class="line">[root<span class="symbol">@localhost</span> examples]<span class="meta"># cd /usr/local/src/</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 安装wget</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta">#yum install wget -y</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 下载HAproxy安装包</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># wget http://www.haproxy.org/download/1.7/src/haproxy-1.7.9.tar.gz</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 解压</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># tar -zxvf haproxy-1.7.9.tar.gz</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 进入目录</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># cd haproxy-1.7.9</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 编译</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># make TARGET=linux2628 PREFIX=/usr/local/haproxy-1.7.9 USE_ZLIB=yes</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 安装</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># make install </span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 拷贝启动文件到目录</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># cp /usr/local/sbin/haproxy /usr/sbin/</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 测试版本</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># haproxy -v</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 拷贝启动文件到启动目录</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># cp examples/haproxy.init /etc/init.d/haproxy</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 赋予可执行权限</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># chmod 755 /etc/init.d/haproxy</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 创建配置文件目录</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># mkdir /etc/haproxy</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 创建数据目录</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># mkdir /var/lib/haproxy</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 创建运行文件目录</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># mkdir /var/run/haproxy</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 设置日志</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># vim /etc/rsyslog.conf</span></span><br><span class="line"><span class="meta"># 第15行  $ModLoad imudp #打开注释</span></span><br><span class="line"><span class="meta"># 第16行  $UDPServerRun 514 #打开注释</span></span><br><span class="line"><span class="meta"># 第74行  local3.* /var/log/haproxy.log #local3的路径</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 创建日志文件</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># touch /var/log/haproxy.log</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 设置权限</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta">#  chown -R haproxy.haproxy /var/log/haproxy.log </span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 启动日志服务</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># systemctl restart rsyslog.service</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="配置HAproxy-Conf："><a href="#配置HAproxy-Conf：" class="headerlink" title="配置HAproxy Conf："></a>配置HAproxy Conf：</h4>
                  <figure class="highlight autoit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># vim /etc/haproxy/haproxy.cfg</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>写入以下内容：</p>
                  <figure class="highlight properties">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># HAProxy 1.7 config for Splash. It assumes Splash instances are executed</span></span><br><span class="line"><span class="comment"># on the same machine and connected to HAProxy using Docker links.</span></span><br><span class="line"><span class="attr">global</span></span><br><span class="line"><span class="comment">    # raise it if necessary</span></span><br><span class="line">    <span class="attr">maxconn</span> <span class="string">512</span></span><br><span class="line"><span class="comment">    # required for stats page</span></span><br><span class="line">    <span class="attr">stats</span> <span class="string">socket /tmp/haproxy</span></span><br><span class="line"></span><br><span class="line"><span class="attr">userlist</span> <span class="string">users</span></span><br><span class="line">    <span class="attr">user</span> <span class="string">user insecure-password userpass</span></span><br><span class="line"></span><br><span class="line"><span class="attr">defaults</span></span><br><span class="line">    <span class="attr">log</span> <span class="string">global</span></span><br><span class="line">    <span class="attr">mode</span> <span class="string">http</span></span><br><span class="line"></span><br><span class="line"><span class="comment">    # remove requests from a queue when clients disconnect;</span></span><br><span class="line"><span class="comment">    # see https://cbonte.github.io/haproxy-dconv/1.7/configuration.html#4.2-option%20abortonclose</span></span><br><span class="line">    <span class="attr">option</span> <span class="string">abortonclose</span></span><br><span class="line"></span><br><span class="line"><span class="comment">    # gzip can save quite a lot of traffic with json, html or base64 data</span></span><br><span class="line"><span class="comment">    # compression algo gzip</span></span><br><span class="line">    <span class="attr">compression</span> <span class="string">type text/html text/plain application/json</span></span><br><span class="line"></span><br><span class="line"><span class="comment">    # increase these values if you want to</span></span><br><span class="line"><span class="comment">    # allow longer request queues in HAProxy</span></span><br><span class="line">    <span class="attr">timeout</span> <span class="string">connect 3600s</span></span><br><span class="line">    <span class="attr">timeout</span> <span class="string">client 3600s</span></span><br><span class="line">    <span class="attr">timeout</span> <span class="string">server 3600s</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># visit 0.0.0.0:8036 to see HAProxy stats page</span></span><br><span class="line"><span class="attr">listen</span> <span class="string">stats</span></span><br><span class="line">    <span class="attr">bind</span> <span class="string">*:8036</span></span><br><span class="line">    <span class="attr">mode</span> <span class="string">http</span></span><br><span class="line">    <span class="attr">stats</span> <span class="string">enable</span></span><br><span class="line">    <span class="attr">stats</span> <span class="string">hide-version</span></span><br><span class="line">    <span class="attr">stats</span> <span class="string">show-legends</span></span><br><span class="line">    <span class="attr">stats</span> <span class="string">show-desc Splash Cluster</span></span><br><span class="line">    <span class="attr">stats</span> <span class="string">uri /</span></span><br><span class="line">    <span class="attr">stats</span> <span class="string">refresh 10s</span></span><br><span class="line">    <span class="attr">stats</span> <span class="string">realm Haproxy\ Statistics</span></span><br><span class="line">    <span class="attr">stats</span> <span class="string">auth    admin:adminpass</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># Splash Cluster configuration</span></span><br><span class="line"><span class="comment"># 代理服务器监听全局的8050端口</span></span><br><span class="line"><span class="attr">frontend</span> <span class="string">http-in</span></span><br><span class="line">    <span class="attr">bind</span> <span class="string">*:8050</span></span><br><span class="line"><span class="comment">    # 如果你需要开启Splash的访问认证</span></span><br><span class="line"><span class="comment">    # 则注释default_backend splash-cluster</span></span><br><span class="line"><span class="comment">    # 并放开其余default_backend splash-cluster 之上的其余注释</span></span><br><span class="line"><span class="comment">    # 账号密码为user  userpass</span></span><br><span class="line"><span class="comment">    # acl auth_ok http_auth(users)</span></span><br><span class="line"><span class="comment">    # http-request auth realm Splash if !auth_ok</span></span><br><span class="line"><span class="comment">    # http-request allow if auth_ok</span></span><br><span class="line"><span class="comment">    # http-request deny</span></span><br><span class="line"></span><br><span class="line"><span class="comment">    # acl staticfiles path_beg /_harviewer/</span></span><br><span class="line"><span class="comment">    # acl misc path / /info /_debug /debug</span></span><br><span class="line"></span><br><span class="line"><span class="comment">    # use_backend splash-cluster if auth_ok !staticfiles !misc</span></span><br><span class="line"><span class="comment">    # use_backend splash-misc if auth_ok staticfiles</span></span><br><span class="line"><span class="comment">    # use_backend splash-misc if auth_ok misc</span></span><br><span class="line">    <span class="attr">default_backend</span> <span class="string">splash-cluster</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="attr">backend</span> <span class="string">splash-cluster</span></span><br><span class="line">    <span class="attr">option</span> <span class="string">httpchk GET /</span></span><br><span class="line">    <span class="attr">balance</span> <span class="string">leastconn</span></span><br><span class="line"></span><br><span class="line"><span class="comment">    # try another instance when connection is dropped</span></span><br><span class="line">    <span class="attr">retries</span> <span class="string">2</span></span><br><span class="line">    <span class="attr">option</span> <span class="string">redispatch</span></span><br><span class="line"><span class="comment">    # 将下面IP地址替换为你自己的Splash服务IP和端口</span></span><br><span class="line"><span class="comment">    # 按照以下格式一次增加其余的Splash服务器</span></span><br><span class="line">    <span class="attr">server</span> <span class="string">splash-0 10.10.1.41:8050 check maxconn 5 inter 2s fall 10 observe layer4</span></span><br><span class="line">    <span class="attr">server</span> <span class="string">splash-1 10.10.1.42:8050 check maxconn 5 inter 2s fall 10 observe layer4</span></span><br><span class="line">    <span class="attr">server</span> <span class="string">splash-2 10.10.1.32:8050 check maxconn 5 inter 2s fall 10 observe layer4</span></span><br><span class="line"></span><br><span class="line"><span class="attr">backend</span> <span class="string">splash-misc</span></span><br><span class="line">    <span class="attr">balance</span> <span class="string">roundrobin</span></span><br><span class="line"><span class="comment">    # 将下面IP地址替换为你自己的Splash服务IP和端口</span></span><br><span class="line"><span class="comment">    # 按照以下格式一次增加其余的Splash服务器</span></span><br><span class="line">    <span class="attr">server</span> <span class="string">splash-0 10.10.1.41:8050 check fall 15</span></span><br><span class="line">    <span class="attr">server</span> <span class="string">splash-1 10.10.1.42:8050 check fall 15</span></span><br><span class="line">    <span class="attr">server</span> <span class="string">splash-2 10.10.1.32:8050 check fall 15</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="启动HAproxy："><a href="#启动HAproxy：" class="headerlink" title="启动HAproxy："></a>启动HAproxy：</h4>
                  <figure class="highlight autoit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta"># 启动HAproxy</span></span><br><span class="line">[root<span class="symbol">@localhost</span> src]<span class="meta"># /etc/init.d/haproxy start</span></span><br><span class="line">Restarting haproxy (via systemctl):                        [  OK  ]</span><br><span class="line"></span><br><span class="line"><span class="meta"># 如果出现错误则使用：</span></span><br><span class="line">[root<span class="symbol">@localhost</span> examples]<span class="meta"># systemctl status haproxy.service</span></span><br><span class="line"></span><br><span class="line"><span class="meta"># 查看报错</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 查看HAproxy状态： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/09/监控面板.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/09/监控面板.png" alt=""></a> 用户名和密码为： admin adminpass</p>
                  <h4 id="查看HAproxy负载是否生效："><a href="#查看HAproxy负载是否生效：" class="headerlink" title="查看HAproxy负载是否生效："></a>查看HAproxy负载是否生效：</h4>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/09/结果.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/09/结果.png" alt=""></a> 完美！！！收工！！ 注意：HAproxy这台服务器没有安装Splash服务，是负载到其余安装有Splash的服务器上提供的服务器哦！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-09-26 16:21:52" itemprop="dateCreated datePublished" datetime="2017-09-26T16:21:52+08:00">2017-09-26</time>
                </span>
                <span id="/4826.html" class="post-meta-item leancloud_visitors" data-flag-title="小白进阶第七篇（Splash负载均衡）" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>6.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>6 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4816.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4816.html" class="post-title-link" itemprop="url">自建免费PYTHON爬虫代理IP池</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>大家好，我还是小四毛，不是崔老师！！！！崔老师在隔壁，哈哈哈。</p>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg" alt=""></a></p>
                  <h2 id="写了一个从网上抓取代理IP，然后构建代理IP池的脚本，放在了这里：https-github-com-xiaosimao-IP-POOL"><a href="#写了一个从网上抓取代理IP，然后构建代理IP池的脚本，放在了这里：https-github-com-xiaosimao-IP-POOL" class="headerlink" title="写了一个从网上抓取代理IP，然后构建代理IP池的脚本，放在了这里：https://github.com/xiaosimao/IP_POOL"></a>写了一个从网上抓取代理IP，然后构建代理IP池的脚本，放在了这里：<a href="https://github.com/xiaosimao/IP_POOL" target="_blank" rel="noopener">https://github.com/xiaosimao/IP_POOL</a></h2>
                  <h2 id="以后应该还会有很多的改动，-欢迎有兴趣的同学star，以便及时可以收到改动的通知。"><a href="#以后应该还会有很多的改动，-欢迎有兴趣的同学star，以便及时可以收到改动的通知。" class="headerlink" title="以后应该还会有很多的改动， 欢迎有兴趣的同学star，以便及时可以收到改动的通知。"></a>以后应该还会有很多的改动， 欢迎有兴趣的同学star，以便及时可以收到改动的通知。</h2>
                  <h2 id="目前是从以下几个网站获取IP：66ip，xicidaili，data5u，proxydb。"><a href="#目前是从以下几个网站获取IP：66ip，xicidaili，data5u，proxydb。" class="headerlink" title="目前是从以下几个网站获取IP：66ip，xicidaili，data5u，proxydb。"></a>目前是从以下几个网站获取IP：66ip，xicidaili，data5u，proxydb。</h2>
                  <h2 id="具体的使用方法文档在readme-md-中，-欢迎交流。"><a href="#具体的使用方法文档在readme-md-中，-欢迎交流。" class="headerlink" title="具体的使用方法文档在readme.md 中， 欢迎交流。"></a>具体的使用方法文档在readme.md 中， 欢迎交流。</h2>
                  <h2 id="如果你需要从别的网站获得，-那么可以在配置文件中进行相关的配置即可，-如果实在不想自己写，也可以提issue给我，我会看情况更新到代码中。"><a href="#如果你需要从别的网站获得，-那么可以在配置文件中进行相关的配置即可，-如果实在不想自己写，也可以提issue给我，我会看情况更新到代码中。" class="headerlink" title="如果你需要从别的网站获得， 那么可以在配置文件中进行相关的配置即可， 如果实在不想自己写，也可以提issue给我，我会看情况更新到代码中。"></a>如果你需要从别的网站获得， 那么可以在配置文件中进行相关的配置即可， 如果实在不想自己写，也可以提issue给我，我会看情况更新到代码中。</h2>
                  <h2 id="一般情况下，只要配置一下配置项就可以从新的网站获取IP，最大限度减少写代码的时间。"><a href="#一般情况下，只要配置一下配置项就可以从新的网站获取IP，最大限度减少写代码的时间。" class="headerlink" title="一般情况下，只要配置一下配置项就可以从新的网站获取IP，最大限度减少写代码的时间。"></a>一般情况下，只要配置一下配置项就可以从新的网站获取IP，最大限度减少写代码的时间。</h2>
                  <h2 id="免费的ip，质量不敢保证，目前测试的目标网站为百度和https-httpbin-org-get，-还是获得了一些通过测试的IP，下面是截图。"><a href="#免费的ip，质量不敢保证，目前测试的目标网站为百度和https-httpbin-org-get，-还是获得了一些通过测试的IP，下面是截图。" class="headerlink" title="免费的ip，质量不敢保证，目前测试的目标网站为百度和https://httpbin.org/get， 还是获得了一些通过测试的IP，下面是截图。"></a>免费的ip，质量不敢保证，目前测试的目标网站为百度和<a href="https://httpbin.org/get，" target="_blank" rel="noopener">https://httpbin.org/get，</a> 还是获得了一些通过测试的IP，下面是截图。</h2>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/09/数据库截图.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/09/数据库截图.png" alt=""></a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/四毛" class="author" itemprop="url" rel="index">四毛</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-09-22 14:47:08" itemprop="dateCreated datePublished" datetime="2017-09-22T14:47:08+08:00">2017-09-22</time>
                </span>
                <span id="/4816.html" class="post-meta-item leancloud_visitors" data-flag-title="自建免费PYTHON爬虫代理IP池" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>390</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4804.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Net <i class="label-arrow"></i>
                  </a>
                  <a href="/4804.html" class="post-title-link" itemprop="url">HTTP 206 获取文件部分内容和范围请求</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>HTTP 2xx 范围内的状态码表明了“客户端发送的请求已经被服务器接受并且被成功处理了”。 HTTP/1.1 200 OK 是 HTTP 请求成功后的标准响应，当你在浏览器中打开某个网站后,你通常会得到一个 200 状态码。HTTP/1.1 206 状态码表示的是“客户端通过发送范围请求头Range抓取到了资源的部分数据” 这种请求通常用来:</p>
                  <ul>
                    <li>学习http头和状态</li>
                    <li>解决网路问题</li>
                    <li>解决大文件下载问题</li>
                    <li>解决CDN和原始HTTP服务器问题</li>
                    <li>使用工具例如lftp,wget,telnet测试断电续传</li>
                    <li>测试将一个大文件分割成多个部分同时下载</li>
                  </ul>
                  <h2 id="测试"><a href="#测试" class="headerlink" title="测试"></a>测试</h2>
                  <p>查看服务器是否支持 HTTP 206：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">curl</span> <span class="string">-I</span> <span class="string">https://raw.githubusercontent.com/Germey/LaravelGeetest/master/README.md</span></span><br><span class="line"><span class="string">HTTP/1.1</span> <span class="number">200</span> <span class="string">OK</span></span><br><span class="line"><span class="attr">Content-Security-Policy:</span> <span class="string">default-src</span> <span class="string">'none'</span><span class="string">;</span> <span class="string">style-src</span> <span class="string">'unsafe-inline'</span></span><br><span class="line"><span class="attr">Strict-Transport-Security:</span> <span class="string">max-age=31536000</span></span><br><span class="line"><span class="attr">X-Content-Type-Options:</span> <span class="string">nosniff</span></span><br><span class="line"><span class="attr">X-Frame-Options:</span> <span class="string">deny</span></span><br><span class="line"><span class="attr">X-XSS-Protection:</span> <span class="number">1</span><span class="string">;</span> <span class="string">mode=block</span></span><br><span class="line"><span class="attr">ETag:</span> <span class="string">"b29f4639b76cd7f94a4b36b05be6c85acfe478f1"</span></span><br><span class="line"><span class="attr">Content-Type:</span> <span class="string">text/plain;</span> <span class="string">charset=utf-8</span></span><br><span class="line"><span class="attr">Cache-Control:</span> <span class="string">max-age=300</span></span><br><span class="line"><span class="attr">X-Geo-Block-List:</span></span><br><span class="line"><span class="attr">X-GitHub-Request-Id:</span> <span class="string">850A:16D2:30128BA:3341504:59BBC946</span></span><br><span class="line"><span class="attr">Content-Length:</span> <span class="number">8709</span></span><br><span class="line"><span class="attr">Accept-Ranges:</span> <span class="string">bytes</span></span><br><span class="line"><span class="attr">Date:</span> <span class="string">Fri,</span> <span class="number">15</span> <span class="string">Sep</span> <span class="number">2017</span> <span class="number">12</span><span class="string">:36:31</span> <span class="string">GMT</span></span><br><span class="line"><span class="attr">Via:</span> <span class="number">1.1</span> <span class="string">varnish</span></span><br><span class="line"><span class="attr">Connection:</span> <span class="string">keep-alive</span></span><br><span class="line"><span class="attr">X-Served-By:</span> <span class="string">cache-nrt6123-NRT</span></span><br><span class="line"><span class="attr">X-Cache:</span> <span class="string">HIT</span></span><br><span class="line"><span class="attr">X-Cache-Hits:</span> <span class="number">1</span></span><br><span class="line"><span class="attr">X-Timer:</span> <span class="string">S1505478991.145862,VS0,VE1</span></span><br><span class="line"><span class="attr">Vary:</span> <span class="string">Authorization,Accept-Encoding</span></span><br><span class="line"><span class="attr">Access-Control-Allow-Origin:</span> <span class="string">*</span></span><br><span class="line"><span class="attr">X-Fastly-Request-ID:</span> <span class="string">ee23d80d2ba507ec0a70c880a075df0d2671aa4d</span></span><br><span class="line"><span class="attr">Expires:</span> <span class="string">Fri,</span> <span class="number">15</span> <span class="string">Sep</span> <span class="number">2017</span> <span class="number">12</span><span class="string">:41:31</span> <span class="string">GMT</span></span><br><span class="line"><span class="attr">Source-Age:</span> <span class="number">8</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中有两个我们比较关注的请求头： Accept-Ranges: bytes：该响应头表明服务器支持 Range 请求，以及服务器所支持的单位是字节。同时服务器支持断点续传，以及支持同时下载文件的多个部分，也就是说下载工具可以利用范围请求加速下载该文件。Accept-Ranges: none 响应头表示服务器不支持范围请求。 Content-Length: 8709 ：Content-Length 响应头表明了响应实体的大小,也就是真实的图片文件的大小是 8709 字节 (8.7K)。</p>
                  <h2 id="发送"><a href="#发送" class="headerlink" title="发送"></a>发送</h2>
                  <p>利用 CURL 可以指定请求范围。 获取前 500 字节：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl --header <span class="string">"Range: bytes=0-500"</span> https:<span class="regexp">//</span>raw.githubusercontent.com<span class="regexp">/Germey/</span>LaravelGeetest<span class="regexp">/master/</span>README.md</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>后 500 字节：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl --header <span class="string">"Range: bytes=-500"</span> https:<span class="regexp">//</span>raw.githubusercontent.com<span class="regexp">/Germey/</span>LaravelGeetest<span class="regexp">/master/</span>README.md</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>从 500 - 1000 字节：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl --header <span class="string">"Range: bytes=500-1000"</span> https:<span class="regexp">//</span>raw.githubusercontent.com<span class="regexp">/Germey/</span>LaravelGeetest<span class="regexp">/master/</span>README.md</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>从 500 - 末尾字节：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl --header <span class="string">"Range: bytes=500-"</span> https:<span class="regexp">//</span>raw.githubusercontent.com<span class="regexp">/Germey/</span>LaravelGeetest<span class="regexp">/master/</span>README.md</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="开启"><a href="#开启" class="headerlink" title="开启"></a>开启</h2>
                  <p>大部分web服务器都原生支持字节范围请求. Apache 2.x用户可以在httpd.conf中尝试 <a href="http://httpd.apache.org/docs/2.2/mod/mod_headers.html" target="_blank" rel="noopener">mod_headers</a>:</p>
                  <figure class="highlight lasso">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Header</span> <span class="built_in">set</span> Accept<span class="params">-Ranges</span> <span class="built_in">bytes</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-09-15 20:46:16" itemprop="dateCreated datePublished" datetime="2017-09-15T20:46:16+08:00">2017-09-15</time>
                </span>
                <span id="/4804.html" class="post-meta-item leancloud_visitors" data-flag-title="HTTP 206 获取文件部分内容和范围请求" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4791.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4791.html" class="post-title-link" itemprop="url">轻型爬虫框架</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="大家好，我是四毛-不是崔老师。"><a href="#大家好，我是四毛-不是崔老师。" class="headerlink" title="大家好，我是四毛,  不是崔老师。"></a>大家好，我是四毛, 不是崔老师。</h1>
                  <h1 id="恩，今天的内容很短-主要都写在了README-md里面。"><a href="#恩，今天的内容很短-主要都写在了README-md里面。" class="headerlink" title="恩，今天的内容很短, 主要都写在了README.md里面。"></a>恩，今天的内容很短, 主要都写在了README.md里面。</h1>
                  <p> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/02/QQ图片20170205084843.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/02/QQ图片20170205084843.jpg" alt=""></a> </p>
                  <h3 id="写了一个将爬虫基本步骤都封装起来的小框架，地址在https-github-com-xiaosimao-AiSpider，-欢迎Star。"><a href="#写了一个将爬虫基本步骤都封装起来的小框架，地址在https-github-com-xiaosimao-AiSpider，-欢迎Star。" class="headerlink" title="写了一个将爬虫基本步骤都封装起来的小框架，地址在https://github.com/xiaosimao/AiSpider， 欢迎Star。"></a><strong>写了一个将爬虫基本步骤都封装起来的小框架，地址在<a href="https://github.com/xiaosimao/AiSpider" target="_blank" rel="noopener">https://github.com/xiaosimao/AiSpider</a>， 欢迎Star。</strong></h3>
                  <h3 id="写的很基础，很简单，大道至简（对，其实就是不会写）。"><a href="#写的很基础，很简单，大道至简（对，其实就是不会写）。" class="headerlink" title="写的很基础，很简单，大道至简（对，其实就是不会写）。"></a><strong>写的很基础，很简单，大道至简（对，其实就是不会写）。</strong></h3>
                  <p><strong>最近也在学一些设计模式的东西。</strong></p>
                  <h3 id="欢迎有兴趣的同学共同研究，readme-md中有我的微信（加的时候麻烦注明一下来自静觅），提出存在的问题和你的想法，这样大家可以共同讨论，共同进步。"><a href="#欢迎有兴趣的同学共同研究，readme-md中有我的微信（加的时候麻烦注明一下来自静觅），提出存在的问题和你的想法，这样大家可以共同讨论，共同进步。" class="headerlink" title="欢迎有兴趣的同学共同研究，readme.md中有我的微信（加的时候麻烦注明一下来自静觅），提出存在的问题和你的想法，这样大家可以共同讨论，共同进步。"></a><strong>欢迎有兴趣的同学共同研究，readme.md中有我的微信（加的时候麻烦注明一下来自静觅），提出存在的问题和你的想法，这样大家可以共同讨论，共同进步。</strong></h3>
                  <h1 id="BE-A-SPIDERMAN。"><a href="#BE-A-SPIDERMAN。" class="headerlink" title="BE A SPIDERMAN。"></a><em><strong>BE A SPIDERMAN。</strong></em></h1>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/四毛" class="author" itemprop="url" rel="index">四毛</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-09-12 17:02:38" itemprop="dateCreated datePublished" datetime="2017-09-12T17:02:38+08:00">2017-09-12</time>
                </span>
                <span id="/4791.html" class="post-meta-item leancloud_visitors" data-flag-title="轻型爬虫框架" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>240</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4778.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4778.html" class="post-title-link" itemprop="url">Neo4j简介及Py2Neo的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>Neo4j是一个世界领先的开源图形数据库，由 Java 编写。图形数据库也就意味着它的数据并非保存在表或集合中，而是保存为节点以及节点之间的关系。 Neo4j 的数据由下面几部分构成：</p>
                  <ul>
                    <li>节点</li>
                    <li>边</li>
                    <li>属性</li>
                  </ul>
                  <p>Neo4j 除了顶点（Node）和边（Relationship），还有一种重要的部分——属性。无论是顶点还是边，都可以有任意多的属性。属性的存放类似于一个 HashMap，Key 为一个字符串，而 Value 必须是基本类型或者是基本类型数组。 在Neo4j中，节点以及边都能够包含保存值的属性，此外：</p>
                  <ul>
                    <li>可以为节点设置零或多个标签（例如 Author 或 Book）</li>
                    <li>每个关系都对应一种类型（例如 WROTE 或 FRIEND_OF）</li>
                    <li>关系总是从一个节点指向另一个节点（但可以在不考虑指向性的情况下进行查询）</li>
                  </ul>
                  <p>具体介绍可以参考：<a href="https://www.w3cschool.cn/neo4j" target="_blank" rel="noopener">https://www.w3cschool.cn/neo4j</a>。</p>
                  <h2 id="Neo4j的特点"><a href="#Neo4j的特点" class="headerlink" title="Neo4j的特点"></a>Neo4j的特点</h2>
                  <ul>
                    <li>它拥有简单的查询语言 Neo4j CQL</li>
                    <li>它遵循属性图数据模型</li>
                    <li>它通过使用 Apache Lucence 支持索引</li>
                    <li>它支持 UNIQUE 约束</li>
                    <li>它包含一个用于执行 CQL 命令的 UI：Neo4j 数据浏览器</li>
                    <li>它支持完整的 ACID（原子性，一致性，隔离性和持久性）规则</li>
                    <li>它采用原生图形库与本地 GPE（图形处理引擎）</li>
                    <li>它支持查询的数据导出到 Json 和 XLS 格式</li>
                    <li>它提供了 REST API，可以被任何编程语言（如 Java，Spring，Scala 等）访问</li>
                    <li>它提供了可以通过任何 UI MVC 框架（如 Node JS ）访问的 Java 脚本</li>
                    <li>它支持两种 Java API：Cypher API 和 Native Java API 来开发 Java 应用程序</li>
                  </ul>
                  <h2 id="Neo4j安装"><a href="#Neo4j安装" class="headerlink" title="Neo4j安装"></a>Neo4j安装</h2>
                  <p>可以到官网直接下载安装包安装即可，链接：<a href="https://neo4j.com/download/" target="_blank" rel="noopener">https://neo4j.com/download/</a>。</p>
                  <h2 id="Neo4j-CQL命令"><a href="#Neo4j-CQL命令" class="headerlink" title="Neo4j CQL命令"></a>Neo4j CQL命令</h2>
                  <p>Neo4j 的 CQL 是非常重要的命令，类似于 SQL 语句，具体的用法可以参考：<a href="https://www.w3cschool.cn/neo4j/neo4j_cql_introduction.html" target="_blank" rel="noopener">https://www.w3cschool.cn/neo4j/neo4j_cql_introduction.html</a>。</p>
                  <h2 id="Py2Neo用法"><a href="#Py2Neo用法" class="headerlink" title="Py2Neo用法"></a>Py2Neo用法</h2>
                  <p>Py2Neo 是用来对接 Neo4j 的 Python 库，接下来对其详细介绍。</p>
                  <h3 id="相关链接"><a href="#相关链接" class="headerlink" title="相关链接"></a>相关链接</h3>
                  <ul>
                    <li>官方文档：<a href="http://py2neo.org/v3/index.html" target="_blank" rel="noopener">http://py2neo.org/v3/index.html</a></li>
                    <li>GitHub：<a href="https://github.com/technige/py2neo" target="_blank" rel="noopener">https://github.com/technige/py2neo</a></li>
                  </ul>
                  <h3 id="安装方法"><a href="#安装方法" class="headerlink" title="安装方法"></a>安装方法</h3>
                  <p>使用 Pip 安装即可：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> py2neo</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="Node-amp-Relationship"><a href="#Node-amp-Relationship" class="headerlink" title="Node &amp; Relationship"></a>Node &amp; Relationship</h3>
                  <p>Neo4j 里面最重要的两个数据结构就是节点和关系，即 Node 和 Relationship，可以通过 Node 或 Relationship 对象创建，实例如下：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from py2neo import <span class="keyword">Node</span><span class="title">, Relationship</span></span><br><span class="line"></span><br><span class="line">a = <span class="keyword">Node</span><span class="title">('Person</span>', <span class="attr">name=</span>'Alice')</span><br><span class="line">b = <span class="keyword">Node</span><span class="title">('Person</span>', <span class="attr">name=</span>'Bob')</span><br><span class="line">r = Relationship(a, 'KNOWS', b)</span><br><span class="line">print(a, b, r)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">alice</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">name</span>:<span class="string">"Alice"</span>&#125;) (<span class="selector-tag">bob</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">name</span>:<span class="string">"Bob"</span>&#125;) (<span class="selector-tag">alice</span>)<span class="selector-tag">-</span><span class="selector-attr">[:KNOWS]</span><span class="selector-tag">-</span>&gt;(<span class="selector-tag">bob</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就成功创建了两个 Node 和两个 Node 之间的 Relationship。 Node 和 Relationship 都继承了 PropertyDict 类，它可以赋值很多属性，类似于字典的形式，例如可以通过如下方式对 Node 或 Relationship 进行属性赋值，接着上面的代码，实例如下:</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">a</span>[<span class="string">'age'</span>] = <span class="number">20</span></span><br><span class="line"><span class="selector-tag">b</span>[<span class="string">'age'</span>] = <span class="number">21</span></span><br><span class="line">r[<span class="string">'time'</span>] = <span class="string">'2017/08/31'</span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(a, b, r)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">alice</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">20</span>,name:<span class="string">"Alice"</span>&#125;) (<span class="selector-tag">bob</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">21</span>,name:<span class="string">"Bob"</span>&#125;) (<span class="selector-tag">alice</span>)<span class="selector-tag">-</span><span class="selector-attr">[:KNOWS &#123;time:<span class="string">"2017/08/31"</span>&#125;]</span><span class="selector-tag">-</span>&gt;(<span class="selector-tag">bob</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见通过类似字典的操作方法就可以成功实现属性赋值。 另外还可以通过 setdefault() 方法赋值默认属性，例如：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">a</span>.setdefault(<span class="string">'location'</span>, <span class="string">'北京'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(a)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">alice</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">20</span>,location:<span class="string">"北京"</span>,name:<span class="string">"Alice"</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见没有给 a 对象赋值 location 属性，现在就会使用默认属性。 但如果赋值了 location 属性，则它会覆盖默认属性，例如：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">a</span>[<span class="string">'location'</span>] = <span class="string">'上海'</span></span><br><span class="line"><span class="selector-tag">a</span>.setdefault(<span class="string">'location'</span>, <span class="string">'北京'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(a)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">alice</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">20</span>,location:<span class="string">"上海"</span>,name:<span class="string">"Alice"</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外也可以使用 update() 方法对属性批量更新，接着上面的例子实例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">data = &#123;</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'Amy'</span>,</span><br><span class="line">    <span class="string">'age'</span>: <span class="number">21</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="selector-tag">a</span>.update(data)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(a)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">alice</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">21</span>,location:<span class="string">"上海"</span>,name:<span class="string">"Amy"</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到这里更新了 a 对象的 name 和 age 属性，没有更新 location 属性，则 name 和 age 属性会更新，location 属性则会保留。</p>
                  <h3 id="Subgraph"><a href="#Subgraph" class="headerlink" title="Subgraph"></a>Subgraph</h3>
                  <p>Subgraph，子图，是 Node 和 Relationship 的集合，最简单的构造子图的方式是通过关系运算符，实例如下：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from py2neo import <span class="keyword">Node</span><span class="title">, Relationship</span></span><br><span class="line"></span><br><span class="line">a = <span class="keyword">Node</span><span class="title">('Person</span>', <span class="attr">name=</span>'Alice')</span><br><span class="line">b = <span class="keyword">Node</span><span class="title">('Person</span>', <span class="attr">name=</span>'Bob')</span><br><span class="line">r = Relationship(a, 'KNOWS', b)</span><br><span class="line">s = a | b | r</span><br><span class="line">print(s)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight clojure">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(&#123;(<span class="name">alice:Person</span> &#123;name:<span class="string">"Alice"</span>&#125;), (<span class="name">bob:Person</span> &#123;name:<span class="string">"Bob"</span>&#125;)&#125;, &#123;(<span class="name">alice</span>)-[<span class="symbol">:KNOWS</span>]-&gt;(<span class="name">bob</span>)&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就组成了一个 Subgraph。 另外还可以通过 nodes() 和 relationships() 方法获取所有的 Node 和 Relationship，实例如下：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">print(<span class="name">s</span>.nodes())</span><br><span class="line">print(<span class="name">s</span>.relationships())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">frozenset(&#123;(<span class="name">alice</span><span class="symbol">:Person</span> &#123;name:<span class="string">"Alice"</span>&#125;), (<span class="name">bob</span><span class="symbol">:Person</span> &#123;name:<span class="string">"Bob"</span>&#125;)&#125;)</span><br><span class="line">frozenset(&#123;(<span class="name">alice</span>)-[<span class="symbol">:KNOWS</span>]-&gt;(<span class="name">bob</span>)&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到结果是 frozenset 类型。 另外还可以利用 &amp; 取 Subgraph 的交集，例如：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">s1 = a <span class="string">| b | r</span></span><br><span class="line">s2 = a <span class="string">| b</span></span><br><span class="line">print(s1 <span class="meta">&amp; s2)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight clojure">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(&#123;(<span class="name">alice:Person</span> &#123;name:<span class="string">"Alice"</span>&#125;), (<span class="name">bob:Person</span> &#123;name:<span class="string">"Bob"</span>&#125;)&#125;, &#123;&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到结果是二者的交集。 另外我们还可以分别利用 keys()、labels()、nodes()、relationships()、types() 分别获取 Subgraph 的 Key、Label、Node、Relationship、Relationship Type，实例如下：</p>
                  <figure class="highlight gauss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">s = a | b | r</span><br><span class="line"><span class="keyword">print</span>(s.<span class="built_in">keys</span>())</span><br><span class="line"><span class="keyword">print</span>(s.<span class="built_in">labels</span>())</span><br><span class="line"><span class="keyword">print</span>(s.<span class="built_in">nodes</span>())</span><br><span class="line"><span class="keyword">print</span>(s.<span class="built_in">relationships</span>())</span><br><span class="line"><span class="keyword">print</span>(s.<span class="built_in">types</span>())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">frozenset(&#123;'name'&#125;)</span><br><span class="line">frozenset(&#123;'Person'&#125;)</span><br><span class="line">frozenset(&#123;(<span class="name">alice</span><span class="symbol">:Person</span> &#123;name:<span class="string">"Alice"</span>&#125;), (<span class="name">bob</span><span class="symbol">:Person</span> &#123;name:<span class="string">"Bob"</span>&#125;)&#125;)</span><br><span class="line">frozenset(&#123;(<span class="name">alice</span>)-[<span class="symbol">:KNOWS</span>]-&gt;(<span class="name">bob</span>)&#125;)</span><br><span class="line">frozenset(&#123;'KNOWS'&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外还可以用 order() 或 size() 方法来获取 Subgraph 的 Node 数量和 Relationship 数量，实例如下：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from py2neo import <span class="keyword">Node</span><span class="title">, Relationship</span>, size, order</span><br><span class="line">s = a | b | r</span><br><span class="line">print(order(s))</span><br><span class="line">print(size(s))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2</span></span><br><span class="line"><span class="number">1</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="Walkable"><a href="#Walkable" class="headerlink" title="Walkable"></a>Walkable</h3>
                  <p>Walkable 是增加了遍历信息的 Subgraph，我们通过 + 号便可以构建一个 Walkable 对象，例如：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from py2neo import <span class="keyword">Node</span><span class="title">, Relationship</span></span><br><span class="line"></span><br><span class="line">a = <span class="keyword">Node</span><span class="title">('Person</span>', <span class="attr">name=</span>'Alice')</span><br><span class="line">b = <span class="keyword">Node</span><span class="title">('Person</span>', <span class="attr">name=</span>'Bob')</span><br><span class="line">c = <span class="keyword">Node</span><span class="title">('Person</span>', <span class="attr">name=</span>'Mike')</span><br><span class="line">ab = Relationship(a, <span class="string">"KNOWS"</span>, b)</span><br><span class="line">ac = Relationship(a, <span class="string">"KNOWS"</span>, c)</span><br><span class="line">w = ab + Relationship(b, <span class="string">"LIKES"</span>, c) + ac</span><br><span class="line">print(w)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight elixir">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(alice)-[<span class="symbol">:KNOWS</span>]-&gt;(bob)-[<span class="symbol">:LIKES</span>]-&gt;(mike)&lt;-[<span class="symbol">:KNOWS</span>]-(alice)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就形成了一个 Walkable 对象。 另外我们可以调用 walk() 方法实现遍历，实例如下：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo import walk</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> <span class="built_in">item</span> <span class="keyword">in</span> walk(w):</span><br><span class="line">    print(<span class="built_in">item</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">alice</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">name</span>:<span class="string">"Alice"</span>&#125;)</span><br><span class="line">(<span class="selector-tag">alice</span>)<span class="selector-tag">-</span><span class="selector-attr">[:KNOWS]</span><span class="selector-tag">-</span>&gt;(<span class="selector-tag">bob</span>)</span><br><span class="line">(<span class="selector-tag">bob</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">name</span>:<span class="string">"Bob"</span>&#125;)</span><br><span class="line">(<span class="selector-tag">bob</span>)<span class="selector-tag">-</span><span class="selector-attr">[:LIKES]</span><span class="selector-tag">-</span>&gt;(<span class="selector-tag">mike</span>)</span><br><span class="line">(<span class="selector-tag">mike</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">name</span>:<span class="string">"Mike"</span>&#125;)</span><br><span class="line">(<span class="selector-tag">alice</span>)<span class="selector-tag">-</span><span class="selector-attr">[:KNOWS]</span><span class="selector-tag">-</span>&gt;(<span class="selector-tag">mike</span>)</span><br><span class="line">(<span class="selector-tag">alice</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">name</span>:<span class="string">"Alice"</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到它从 a 这个 Node 开始遍历，然后到 b，再到 c，最后重新回到 a。 另外还可以利用 start_node()、end_node()、nodes()、relationships() 方法来获取起始 Node、终止 Node、所有 Node 和 Relationship，例如：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">print(<span class="name">w</span>.start_node())</span><br><span class="line">print(<span class="name">w</span>.end_node())</span><br><span class="line">print(<span class="name">w</span>.nodes())</span><br><span class="line">print(<span class="name">w</span>.relationships())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight clojure">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="name">alice:Person</span> &#123;name:<span class="string">"Alice"</span>&#125;)</span><br><span class="line">(<span class="name">alice:Person</span> &#123;name:<span class="string">"Alice"</span>&#125;)</span><br><span class="line">((<span class="name">alice:Person</span> &#123;name:<span class="string">"Alice"</span>&#125;), (<span class="name">bob:Person</span> &#123;name:<span class="string">"Bob"</span>&#125;), (<span class="name">mike:Person</span> &#123;name:<span class="string">"Mike"</span>&#125;), (<span class="name">alice:Person</span> &#123;name:<span class="string">"Alice"</span>&#125;))</span><br><span class="line">((<span class="name">alice</span>)-[<span class="symbol">:KNOWS</span>]-&gt;(<span class="name">bob</span>), (<span class="name">bob</span>)-[<span class="symbol">:LIKES</span>]-&gt;(<span class="name">mike</span>), (<span class="name">alice</span>)-[<span class="symbol">:KNOWS</span>]-&gt;(<span class="name">mike</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到本例中起始和终止 Node 都是同一个，这和 walk() 方法得到的结果是一致的。</p>
                  <h3 id="Graph"><a href="#Graph" class="headerlink" title="Graph"></a>Graph</h3>
                  <p>在 database 模块中包含了和 Neo4j 数据交互的 API，最重要的当属 Graph，它代表了 Neo4j 的图数据库，同时 Graph 也提供了许多方法来操作 Neo4j 数据库。 Graph 在初始化的时候需要传入连接的 URI，初始化参数有 bolt、secure、host、http_port、https_port、bolt_port、user、password，详情说明可以参考：<a href="http://py2neo.org/v3/database.html#py2neo.database.Graph" target="_blank" rel="noopener">http://py2neo.org/v3/database.html#py2neo.database.Graph</a>。 初始化的实例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">py2neo</span> <span class="variable">import</span> <span class="variable">Graph</span></span><br><span class="line"><span class="variable">graph_1</span> = <span class="function"><span class="title">Graph</span>()</span></span><br><span class="line"><span class="variable">graph_2</span> = <span class="function"><span class="title">Graph</span>(<span class="variable">host</span>=<span class="string">"localhost"</span>)</span></span><br><span class="line"><span class="variable">graph_3</span> = <span class="function"><span class="title">Graph</span>(<span class="string">"http://localhost:7474/db/data/"</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外我们还可以利用 create() 方法传入 Subgraph 对象来将关系图添加到数据库中，实例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo import Node, Relationship, Graph</span><br><span class="line"></span><br><span class="line">a = Node(<span class="string">'Person'</span>, <span class="attribute">name</span>=<span class="string">'Alice'</span>)</span><br><span class="line">b = Node(<span class="string">'Person'</span>, <span class="attribute">name</span>=<span class="string">'Bob'</span>)</span><br><span class="line">r = Relationship(a, <span class="string">'KNOWS'</span>, b)</span><br><span class="line">s = a | b | r</span><br><span class="line">graph = Graph(<span class="attribute">password</span>=<span class="string">'123456'</span>)</span><br><span class="line">graph.create(s)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里必须确保 Neo4j 正常运行，其密码为 123456，这里调用 create() 方法即可完成图的创建，结果如下： <img src="https://germey.gitbooks.io/ai/content/assets/2017-08-31-21-35-08.jpg" alt=""> 另外我们也可以单独添加单个 Node 或 Relationship，实例如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo <span class="keyword">import</span> Graph, Node, Relationship</span><br><span class="line"></span><br><span class="line">graph = Graph(<span class="keyword">password</span>=<span class="string">'123456'</span>)</span><br><span class="line">a = Node(<span class="string">'Person'</span>, <span class="type">name</span>=<span class="string">'Alice'</span>)</span><br><span class="line">graph.<span class="keyword">create</span>(a)</span><br><span class="line">b = Node(<span class="string">'Person'</span>, <span class="type">name</span>=<span class="string">'Bob'</span>)</span><br><span class="line">ab = Relationship(a, <span class="string">'KNOWS'</span>, b)</span><br><span class="line">graph.<span class="keyword">create</span>(ab)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下： <img src="https://germey.gitbooks.io/ai/content/assets/2017-08-31-22-00-02.jpg" alt=""> 另外还可以利用 data() 方法来获取查询结果：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="title">from</span> py2neo <span class="keyword">import</span> Graph</span><br><span class="line"></span><br><span class="line"><span class="title">graph</span> = <span class="type">Graph</span>(password='<span class="number">123456</span>')</span><br><span class="line"><span class="class"><span class="keyword">data</span> = graph.<span class="keyword">data</span>('<span class="type">MATCH</span> (<span class="title">p</span>:<span class="type">Person</span>) return p')</span></span><br><span class="line"><span class="title">print</span>(<span class="class"><span class="keyword">data</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight clojure">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&#123;'p': (<span class="name">e0d0f96:Person</span> &#123;name:<span class="string">"Alice"</span>&#125;)&#125;, &#123;'p': (<span class="name">cfe57d0:Person</span> &#123;name:<span class="string">"Bob"</span>&#125;)&#125;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里是通过 CQL 语句实现的查询，输出结果即 CQL 语句的返回结果，是列表形式。 另外输出结果还可以直接转化为 DataFrame 对象，实例如下：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="title">from</span> py2neo <span class="keyword">import</span> Graph</span><br><span class="line"><span class="title">from</span> pandas <span class="keyword">import</span> DataFrame</span><br><span class="line"><span class="title">graph</span> = <span class="type">Graph</span>(password='<span class="number">123456</span>')</span><br><span class="line"><span class="class"><span class="keyword">data</span> = graph.<span class="keyword">data</span>('<span class="type">MATCH</span> (<span class="title">p</span>:<span class="type">Person</span>) return p')</span></span><br><span class="line"><span class="title">df</span> = <span class="type">DataFrame</span>(<span class="class"><span class="keyword">data</span>)</span></span><br><span class="line"><span class="title">print</span>(df)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">                   p</span><br><span class="line"><span class="number">0</span>  &#123;<span class="string">'name'</span>: <span class="string">'Alice'</span>&#125;</span><br><span class="line"><span class="number">1</span>    &#123;<span class="string">'name'</span>: <span class="string">'Bob'</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外可以使用 find_one() 或 find() 方法进行 Node 的查找，可以利用 match() 或 match_one() 方法对 Relationship 进行查找：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo import Graph</span><br><span class="line"></span><br><span class="line">graph = Graph(<span class="attribute">password</span>=<span class="string">'123456'</span>)</span><br><span class="line">node = graph.find_one(<span class="attribute">label</span>=<span class="string">'Person'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(node)</span><br><span class="line">relationship = graph.match_one(<span class="attribute">rel_type</span>=<span class="string">'KNOWS'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(relationship)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">c7402c7</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">21</span>,name:<span class="string">"Alice"</span>&#125;)</span><br><span class="line">(<span class="selector-tag">c7402c7</span>)<span class="selector-tag">-</span><span class="selector-attr">[:KNOWS]</span><span class="selector-tag">-</span>&gt;(<span class="selector-tag">e2c42fc</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果想要更新 Node 的某个属性可以使用 push() 方法，例如：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from py2neo import Graph, <span class="keyword">Node</span><span class="title"></span></span><br><span class="line"><span class="title"></span></span><br><span class="line"><span class="title">graph</span> = Graph(<span class="attr">password=</span>'<span class="number">123456</span>')</span><br><span class="line">a = <span class="keyword">Node</span><span class="title">('Person</span>', <span class="attr">name=</span>'Alice')</span><br><span class="line"><span class="keyword">node</span> <span class="title">= graph</span>.find_one(<span class="attr">label=</span>'Person')</span><br><span class="line"><span class="keyword">node</span><span class="title">['age</span>'] = <span class="number">21</span></span><br><span class="line">graph.push(<span class="keyword">node</span><span class="title">)</span></span><br><span class="line"><span class="title">print</span>(graph.find_one(<span class="attr">label=</span>'Person'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">a90a763</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">21</span>,name:<span class="string">"Alice"</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果想要删除某个 Node 可以使用 delete() 方法，例如：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo import Graph</span><br><span class="line"></span><br><span class="line">graph = Graph(<span class="attribute">password</span>=<span class="string">'123456'</span>)</span><br><span class="line">node = graph.find_one(<span class="attribute">label</span>=<span class="string">'Person'</span>)</span><br><span class="line">relationship = graph.match_one(<span class="attribute">rel_type</span>=<span class="string">'KNOWS'</span>)</span><br><span class="line">graph.delete(relationship)</span><br><span class="line">graph.delete(node)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在删除 Node 时必须先删除其对应的 Relationship，否则无法删除 Node。 另外我们也可以通过 run() 方法直接执行 CQL 语句，例如：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="title">from</span> py2neo <span class="keyword">import</span> Graph</span><br><span class="line"></span><br><span class="line"><span class="title">graph</span> = <span class="type">Graph</span>(password='<span class="number">123456</span>')</span><br><span class="line"><span class="class"><span class="keyword">data</span> = graph.run('<span class="type">MATCH</span> (<span class="title">p</span>:<span class="type">Person</span>) <span class="type">RETURN</span> p <span class="type">LIMIT</span> 5')</span></span><br><span class="line"><span class="title">print</span>(list(<span class="class"><span class="keyword">data</span>))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight clojure">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[(<span class="name">'p':</span> (<span class="name">b6f61ff:Person</span> &#123;age:20,name:<span class="string">"Alice"</span>&#125;)), (<span class="name">'p':</span> (<span class="name">cc238b1:Person</span> &#123;age:20,name:<span class="string">"Alice"</span>&#125;)), (<span class="name">'p':</span> (<span class="name">b09e672:Person</span> &#123;age:20,name:<span class="string">"Alice"</span>&#125;))]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="NodeSelector"><a href="#NodeSelector" class="headerlink" title="NodeSelector"></a>NodeSelector</h3>
                  <p>Graph 有时候用起来不太方便，比如如果要根据多个条件进行 Node 的查询是做不到的，在这里更方便的查询方法是利用 NodeSelector，我们首先新建如下的 Node 和 Relationship，实例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo import Graph, Node, Relationship</span><br><span class="line"></span><br><span class="line">graph = Graph(<span class="attribute">password</span>=<span class="string">'123456'</span>)</span><br><span class="line">a = Node(<span class="string">'Person'</span>, <span class="attribute">name</span>=<span class="string">'Alice'</span>, <span class="attribute">age</span>=21, <span class="attribute">location</span>=<span class="string">'广州'</span>)</span><br><span class="line">b = Node(<span class="string">'Person'</span>, <span class="attribute">name</span>=<span class="string">'Bob'</span>, <span class="attribute">age</span>=22, <span class="attribute">location</span>=<span class="string">'上海'</span>)</span><br><span class="line">c = Node(<span class="string">'Person'</span>, <span class="attribute">name</span>=<span class="string">'Mike'</span>, <span class="attribute">age</span>=21, <span class="attribute">location</span>=<span class="string">'北京'</span>)</span><br><span class="line">r1 = Relationship(a, <span class="string">'KNOWS'</span>, b)</span><br><span class="line">r2 = Relationship(b, <span class="string">'KNOWS'</span>, c)</span><br><span class="line">graph.create(a)</span><br><span class="line">graph.create(r1)</span><br><span class="line">graph.create(r2)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果： <img src="https://germey.gitbooks.io/ai/content/assets/2017-08-31-23-38-27.jpg" alt=""> 在这里我们用 NodeSelector 来筛选 age 为 21 的 Person Node，实例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo import Graph, NodeSelector</span><br><span class="line"></span><br><span class="line">graph = Graph(<span class="attribute">password</span>=<span class="string">'123456'</span>)</span><br><span class="line">selector = NodeSelector(graph)</span><br><span class="line">persons = selector.select(<span class="string">'Person'</span>, <span class="attribute">age</span>=21)</span><br><span class="line"><span class="builtin-name">print</span>(list(persons))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[(<span class="string">d195b2e:</span>Person &#123;<span class="string">age:</span><span class="number">21</span>,<span class="string">location:</span><span class="string">"广州"</span>,<span class="string">name:</span><span class="string">"Alice"</span>&#125;), (<span class="string">eefe475:</span>Person &#123;<span class="string">age:</span><span class="number">21</span>,<span class="string">location:</span><span class="string">"北京"</span>,<span class="string">name:</span><span class="string">"Mike"</span>&#125;)]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外也可以使用 where() 进行更复杂的查询，例如查找 name 是 A 开头的 Person Node，实例如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo <span class="keyword">import</span> Graph, NodeSelector</span><br><span class="line"></span><br><span class="line">graph = Graph(<span class="keyword">password</span>=<span class="string">'123456'</span>)</span><br><span class="line">selector = NodeSelector(graph)</span><br><span class="line">persons = selector.<span class="keyword">select</span>(<span class="string">'Person'</span>).<span class="keyword">where</span>(<span class="string">'_.name =~ "A.*"'</span>)</span><br><span class="line">print(list(persons))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight clojure">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[(<span class="name">bcd8072:Person</span> &#123;age:21,location:<span class="string">"广州"</span>,name:<span class="string">"Alice"</span>&#125;)]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里用了正则表达式匹配查询。 另外也可以使用 order_by() 进行排序：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from py2neo import Graph, NodeSelector</span><br><span class="line"></span><br><span class="line">graph = <span class="constructor">Graph(<span class="params">password</span>='123456')</span></span><br><span class="line">selector = <span class="constructor">NodeSelector(<span class="params">graph</span>)</span></span><br><span class="line">persons = selector.select('Person').order<span class="constructor">_by('<span class="params">_</span>.<span class="params">age</span>')</span></span><br><span class="line">print(<span class="built_in">list</span>(persons))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[(<span class="string">e3fc3d7:</span>Person &#123;<span class="string">age:</span><span class="number">21</span>,<span class="string">location:</span><span class="string">"广州"</span>,<span class="string">name:</span><span class="string">"Alice"</span>&#125;), (<span class="string">da0179d:</span>Person &#123;<span class="string">age:</span><span class="number">21</span>,<span class="string">location:</span><span class="string">"北京"</span>,<span class="string">name:</span><span class="string">"Mike"</span>&#125;), (<span class="string">cafa16e:</span>Person &#123;<span class="string">age:</span><span class="number">22</span>,<span class="string">location:</span><span class="string">"上海"</span>,<span class="string">name:</span><span class="string">"Bob"</span>&#125;)]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>前面返回的都是列表，如果要查询单个节点的话，可以使用 first() 方法，实例如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo <span class="keyword">import</span> Graph, NodeSelector</span><br><span class="line"></span><br><span class="line">graph = Graph(<span class="keyword">password</span>=<span class="string">'123456'</span>)</span><br><span class="line">selector = NodeSelector(graph)</span><br><span class="line">person = selector.<span class="keyword">select</span>(<span class="string">'Person'</span>).<span class="keyword">where</span>(<span class="string">'_.name =~ "A.*"'</span>).first()</span><br><span class="line">print(person)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">ea81c04</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">21</span>,location:<span class="string">"广州"</span>,name:<span class="string">"Alice"</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>更详细的内容可以查看：<a href="http://py2neo.org/v3/database.html#cypher-utilities" target="_blank" rel="noopener">http://py2neo.org/v3/database.html#cypher-utilities</a>。</p>
                  <h3 id="OGM"><a href="#OGM" class="headerlink" title="OGM"></a>OGM</h3>
                  <p>OGM 类似于 ORM，意为 Object Graph Mapping，这样可以实现一个对象和 Node 的关联，例如：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo.ogm <span class="keyword">import</span> GraphObject, Property, RelatedTo, RelatedFrom</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="symbol">Movie</span>(<span class="symbol">GraphObject</span>):</span><br><span class="line">    <span class="symbol">__primarykey__</span> = '<span class="symbol">title</span>'</span><br><span class="line"></span><br><span class="line">    <span class="symbol">title</span> = <span class="symbol">Property</span>()</span><br><span class="line">    <span class="symbol">released</span> = <span class="symbol">Property</span>()</span><br><span class="line">    <span class="symbol">actors</span> = <span class="symbol">RelatedFrom</span>('<span class="symbol">Person</span>', '<span class="symbol">ACTED_IN</span>')</span><br><span class="line">    <span class="symbol">directors</span> = <span class="symbol">RelatedFrom</span>('<span class="symbol">Person</span>', '<span class="symbol">DIRECTED</span>')</span><br><span class="line">    <span class="symbol">producers</span> = <span class="symbol">RelatedFrom</span>('<span class="symbol">Person</span>', '<span class="symbol">PRODUCED</span>')</span><br><span class="line"></span><br><span class="line"><span class="symbol">class</span> <span class="symbol">Person</span>(<span class="symbol">GraphObject</span>):</span><br><span class="line">    <span class="symbol">__primarykey__</span> = '<span class="symbol">name</span>'</span><br><span class="line"></span><br><span class="line">    <span class="symbol">name</span> = <span class="symbol">Property</span>()</span><br><span class="line">    <span class="symbol">born</span> = <span class="symbol">Property</span>()</span><br><span class="line">    <span class="symbol">acted_in</span> = <span class="symbol">RelatedTo</span>('<span class="symbol">Movie</span>')</span><br><span class="line">    <span class="symbol">directed</span> = <span class="symbol">RelatedTo</span>('<span class="symbol">Movie</span>')</span><br><span class="line">    <span class="symbol">produced</span> = <span class="symbol">RelatedTo</span>('<span class="symbol">Movie</span>')</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们可以用它来结合 Graph 查询，例如：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo import Graph</span><br><span class="line"><span class="keyword">from</span> py2neo.ogm import GraphObject, Property</span><br><span class="line"></span><br><span class="line">graph = Graph(<span class="attribute">password</span>=<span class="string">'123456'</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">class Person(GraphObject):</span><br><span class="line">    __primarykey__ = <span class="string">'name'</span></span><br><span class="line"></span><br><span class="line">    name = Property()</span><br><span class="line">    age = Property()</span><br><span class="line">    location = Property()</span><br><span class="line"></span><br><span class="line">person = Person.select(graph).where(<span class="attribute">age</span>=21).first()</span><br><span class="line"><span class="builtin-name">print</span>(person)</span><br><span class="line"><span class="builtin-name">print</span>(person.name)</span><br><span class="line"><span class="builtin-name">print</span>(person.age)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;Person <span class="attribute">name</span>=<span class="string">'Alice'</span>&gt;</span><br><span class="line">Alice</span><br><span class="line">21</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就成功实现了对象和 Node 的映射。 我们可以用它动态改变 Node 的属性，例如修改某个 Node 的 age 属性，实例如下：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">person = Person.select(graph).where(<span class="attr">age=</span><span class="number">21</span>).first()</span><br><span class="line">print(person.__ogm__.<span class="keyword">node</span><span class="title">)</span></span><br><span class="line"><span class="title">person</span>.age = <span class="number">22</span></span><br><span class="line">print(person.__ogm__.<span class="keyword">node</span><span class="title">)</span></span><br><span class="line"><span class="title">graph</span>.push(person)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="selector-tag">ccf5640</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">21</span>,location:<span class="string">"北京"</span>,name:<span class="string">"Mike"</span>&#125;)</span><br><span class="line">(<span class="selector-tag">ccf5640</span><span class="selector-pseudo">:Person</span> &#123;<span class="attribute">age</span>:<span class="number">22</span>,location:<span class="string">"北京"</span>,name:<span class="string">"Mike"</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外我们也可以通过映射关系进行 Relationship 的调整，例如通过 Relationship 添加一个关联 Node，实例如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> py2neo <span class="keyword">import</span> Graph</span><br><span class="line"><span class="keyword">from</span> py2neo.ogm <span class="keyword">import</span> GraphObject, Property, RelatedTo</span><br><span class="line"></span><br><span class="line">graph = Graph(<span class="keyword">password</span>=<span class="string">'123456'</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> Person(GraphObject):</span><br><span class="line">    __primarykey__ = <span class="string">'name'</span></span><br><span class="line"></span><br><span class="line">    <span class="type">name</span> = Property()</span><br><span class="line">    age = Property()</span><br><span class="line">    <span class="keyword">location</span> = Property()</span><br><span class="line">    knows = RelatedTo(<span class="string">'Person'</span>, <span class="string">'KNOWS'</span>)</span><br><span class="line"></span><br><span class="line">person = Person.<span class="keyword">select</span>(graph).<span class="keyword">where</span>(age=<span class="number">21</span>).first()</span><br><span class="line">print(list(person.knows))</span><br><span class="line">new_person = Person()</span><br><span class="line">new_person.name = <span class="string">'Durant'</span></span><br><span class="line">new_person.age = <span class="number">28</span></span><br><span class="line">person.knows.<span class="keyword">add</span>(new_person)</span><br><span class="line">print(list(person.knows))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">[&lt;Person name='Bob'&gt;]</span></span><br><span class="line"><span class="meta">[&lt;Person name='Bob'&gt;, &lt;Person name='Durant'&gt;]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就完成了 Node 和 Relationship 的添加，同时由于设置了 primarykey 为 name，所以不会重复添加。 但是注意此时数据库并没有更新，只是对象更新了，如果要更新到数据库中还需要调用 Graph 对象的 push() 或 pull() 方法，添加如下代码即可：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">graph</span><span class="selector-class">.push</span>(<span class="selector-tag">person</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>也可以通过 remove() 方法移除某个关联 Node，实例如下：</p>
                  <figure class="highlight fortran">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">person = Person.<span class="keyword">select</span>(graph).<span class="keyword">where</span>(<span class="keyword">name</span>=<span class="string">'Alice'</span>).first()</span><br><span class="line"><span class="keyword">target</span> = Person.<span class="keyword">select</span>(graph).<span class="keyword">where</span>(<span class="keyword">name</span>=<span class="string">'Durant'</span>).first()</span><br><span class="line">person.knows.remove(<span class="keyword">target</span>)</span><br><span class="line">graph.push(person)</span><br><span class="line">graph.delete(<span class="keyword">target</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 target 是 name 为 Durant 的 Node，代码运行完毕后即可删除关联 Relationship 和删除 Node。 以上便是 OGM 的用法，查询修改非常方便，推荐使用此方法进行 Node 和 Relationship 的修改。 更多内容可以查看：<a href="http://py2neo.org/v3/ogm.html#module-py2neo.ogm" target="_blank" rel="noopener">http://py2neo.org/v3/ogm.html#module-py2neo.ogm</a>。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>以上便是对 Neo4j 的相关介绍。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-09-01 01:29:20" itemprop="dateCreated datePublished" datetime="2017-09-01T01:29:20+08:00">2017-09-01</time>
                </span>
                <span id="/4778.html" class="post-meta-item leancloud_visitors" data-flag-title="Neo4j简介及Py2Neo的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>12k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>11 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4759.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4759.html" class="post-title-link" itemprop="url">记scikit-learn贝叶斯文本分类的坑（弄了个笨办法解决了，有其它办法的小哥儿请指点）</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>基本步骤： 1、训练素材分类： 我是参考官方的目录结构： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/08/s01.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/08/s01.png" alt=""></a> 每个目录中放对应的文本，一个txt文件一篇对应的文章：就像下面这样 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/08/s02.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/08/s02.png" alt=""></a> 需要注意的是所有素材比例请保持在相同的比例（根据训练结果酌情调整、不可比例过于悬殊、容易造成过拟合（通俗点就是大部分文章都给你分到素材最多的那个类别去了）） 废话不多说直接上代码吧（测试代码的丑得一逼；将就着看看吧） 需要一个小工具： pip install chinese-tokenizer 这是训练器：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> jieba</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">from</span> io <span class="keyword">import</span> BytesIO</span><br><span class="line"><span class="keyword">from</span> chinese_tokenizer.tokenizer <span class="keyword">import</span> Tokenizer</span><br><span class="line"><span class="keyword">from</span> sklearn.datasets <span class="keyword">import</span> load_files</span><br><span class="line"><span class="keyword">from</span> sklearn.feature_extraction.text <span class="keyword">import</span> CountVectorizer, TfidfTransformer</span><br><span class="line"><span class="keyword">from</span> sklearn.model_selection <span class="keyword">import</span> train_test_split</span><br><span class="line"><span class="keyword">from</span> sklearn.naive_bayes <span class="keyword">import</span> MultinomialNB</span><br><span class="line"><span class="keyword">from</span> sklearn.externals <span class="keyword">import</span> joblib</span><br><span class="line"></span><br><span class="line">jie_ba_tokenizer = Tokenizer().jie_ba_tokenizer</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载数据集</span></span><br><span class="line">training_data = load_files(<span class="string">'./data'</span>, encoding=<span class="string">'utf-8'</span>)</span><br><span class="line"><span class="comment"># x_train txt内容 y_train 是类别（正 负 中 ）</span></span><br><span class="line">x_train, _, y_train, _ = train_test_split(training_data.data, training_data.target)</span><br><span class="line">print(<span class="string">'开始建模.....'</span>)</span><br><span class="line"><span class="keyword">with</span> open(<span class="string">'training_data.target'</span>, <span class="string">'w'</span>, encoding=<span class="string">'utf-8'</span>) <span class="keyword">as</span> f:</span><br><span class="line">    f.write(json.dumps(training_data.target_names))</span><br><span class="line"><span class="comment"># tokenizer参数是用来对文本进行分词的函数（就是上面我们结巴分词）</span></span><br><span class="line">count_vect = CountVectorizer(tokenizer=jieba_tokenizer)</span><br><span class="line"></span><br><span class="line">tfidf_transformer = TfidfTransformer()</span><br><span class="line">X_train_counts = count_vect.fit_transform(x_train)</span><br><span class="line"></span><br><span class="line">X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)</span><br><span class="line">print(<span class="string">'正在训练分类器.....'</span>)</span><br><span class="line"><span class="comment"># 多项式贝叶斯分类器训练</span></span><br><span class="line">clf = MultinomialNB().fit(X_train_tfidf, y_train)</span><br><span class="line"><span class="comment"># 保存分类器（好在其它程序中使用）</span></span><br><span class="line">joblib.dump(clf, <span class="string">'model.pkl'</span>)</span><br><span class="line"><span class="comment"># 保存矢量化（坑在这儿！！需要使用和训练器相同的 矢量器 不然会报错！！！！！！ 提示 ValueError dimension mismatch··）</span></span><br><span class="line">joblib.dump(count_vect, <span class="string">'count_vect'</span>)</span><br><span class="line">print(<span class="string">"分类器的相关信息："</span>)</span><br><span class="line">print(clf)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>下面是是使用训练好的分类器分类文章： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/08/s03.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/08/s03.png" alt=""></a> 需要分类的文章放在predict_data目录中：照样是一篇文章一个txt文件</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># -*- coding: utf<span class="number">-8</span> -*-</span><br><span class="line"># @<span class="type">Time</span>    : <span class="number">2017</span>/<span class="number">8</span>/<span class="number">23</span> <span class="number">18</span>:<span class="number">02</span></span><br><span class="line"># @Author  : 哎哟卧槽</span><br><span class="line"># @Site    : </span><br><span class="line"># @File    : 贝叶斯分类器.py</span><br><span class="line"># @Software: PyCharm</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> jieba</span><br><span class="line"><span class="keyword">import</span> <span class="type">json</span></span><br><span class="line"><span class="keyword">from</span> sklearn.datasets <span class="keyword">import</span> load_files</span><br><span class="line"><span class="keyword">from</span> sklearn.feature_extraction.text <span class="keyword">import</span> CountVectorizer, TfidfTransformer</span><br><span class="line"><span class="keyword">from</span> sklearn.externals <span class="keyword">import</span> joblib</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># 加载分类器</span><br><span class="line">clf = joblib.<span class="keyword">load</span>(<span class="string">'model.pkl'</span>)</span><br><span class="line"></span><br><span class="line">count_vect = joblib.<span class="keyword">load</span>(<span class="string">'count_vect'</span>)</span><br><span class="line">testing_data = load_files(<span class="string">'./predict_data'</span>, encoding=<span class="string">'utf-8'</span>)</span><br><span class="line">target_names = <span class="type">json</span>.loads(<span class="keyword">open</span>(<span class="string">'training_data.target'</span>, <span class="string">'r'</span>, encoding=<span class="string">'utf-8'</span>).<span class="keyword">read</span>())</span><br><span class="line">#     # 字符串处理</span><br><span class="line">tfidf_transformer = TfidfTransformer()</span><br><span class="line"></span><br><span class="line">X_new_counts = count_vect.<span class="keyword">transform</span>(testing_data.data)</span><br><span class="line">X_new_tfidf = tfidf_transformer.fit_transform(X_new_counts)</span><br><span class="line"># 进行预测</span><br><span class="line">predicted = clf.predict(X_new_tfidf)</span><br><span class="line"><span class="keyword">for</span> title, category <span class="keyword">in</span> zip(testing_data.filenames, predicted):</span><br><span class="line">    print(<span class="string">'%r =&gt; %s'</span> % (title, target_names[category]))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 这个样子将训练好的分类器在新的程序中使用时候 就不报错： ValueError dimension mismatch·· 这儿有个demo 仅供参考：<a href="https://github.com/thsheep/sklearn_naive_bayes_classification" target="_blank" rel="noopener">GitHub地址</a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-08-23 18:49:44" itemprop="dateCreated datePublished" datetime="2017-08-23T18:49:44+08:00">2017-08-23</time>
                </span>
                <span id="/4759.html" class="post-meta-item leancloud_visitors" data-flag-title="记scikit-learn贝叶斯文本分类的坑（弄了个笨办法解决了，有其它办法的小哥儿请指点）" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.4k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4725.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4725.html" class="post-title-link" itemprop="url">小白进阶之Scrapy第五篇（Scrapy-Splash配合CrawlSpider；瞎几把整的）</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>估摸着各位小伙伴儿被想使用CrawlSpider的Rule来抓取JS，相当受折磨； CrawlSpider Rule总是不能和Splash结合。 废话不多说，手疼···· </p>
                  <h1 id="方法1："><a href="#方法1：" class="headerlink" title="方法1："></a><strong>方法1：</strong></h1>
                  <p>写一个自定义的函数，使用Rule中的process_request参数；来替换掉Rule本身Request的逻辑。 参考官方文档： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/07/QQ20170712-002911.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/07/QQ20170712-002911.png" alt=""></a> 1、将请求更换为SplashRequest请求： 2、每次请求将本次请求的URL使用Meta参数传递下去； 3、重写 _requests_to_follow 方法：替换响应Response的URL为我们传递的URL（否则会格式为Splash的地址） 就像下面这样</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MySpider</span><span class="params">(CrawlSpider)</span>:</span></span><br><span class="line"></span><br><span class="line">    name = <span class="string">'innda'</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">start_requests</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="keyword">yield</span> SplashRequest(url, dont_process_response=<span class="literal">True</span>, args=&#123;<span class="string">'wait'</span>: <span class="number">0.5</span>&#125;, meta=&#123;<span class="string">'real_url'</span>: url&#125;)</span><br><span class="line">        </span><br><span class="line">    rules = (</span><br><span class="line">        Rule(LinkExtractor(allow=(<span class="string">'node_\d+\.htm'</span>,)), process_request=<span class="string">'splash_request'</span>, follow=<span class="literal">True</span>),</span><br><span class="line">        Rule(LinkExtractor(allow=(<span class="string">'content_\d+\.htm'</span>,)), callback=<span class="string">"one_parse"</span>)</span><br><span class="line">    )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">splash_request</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        :param request: Request对象（是一个字典；怎么取值就不说了吧！！）</span></span><br><span class="line"><span class="string">        :return: SplashRequest的请求</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="comment"># dont_process_response=True 参数表示不更改响应对象类型（默认为：HTMLResponse；更改后为：SplashTextResponse）</span></span><br><span class="line">        <span class="comment"># args=&#123;'wait': 0.5&#125; 表示传递等待参数0.5（Splash会渲染0.5s的时间）</span></span><br><span class="line">        <span class="comment"># meta 传递请求的当前请求的URL</span></span><br><span class="line">        <span class="keyword">return</span> SplashRequest(url=request.url, dont_process_response=<span class="literal">True</span>, args=&#123;<span class="string">'wait'</span>: <span class="number">0.5</span>&#125;, meta=&#123;<span class="string">'real_url'</span>: request.url&#125;)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_requests_to_follow</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        <span class="string">"""重写的函数哈！这个函数是Rule的一个方法</span></span><br><span class="line"><span class="string">        :param response: 这货是啥看名字都知道了吧（这货也是个字典，然后你懂的ｄ(･∀･*)♪ﾟ）</span></span><br><span class="line"><span class="string">        :return: 追踪的Request</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> isinstance(response, HtmlResponse):</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line">        seen = set()</span><br><span class="line">        <span class="comment"># 将Response的URL更改为我们传递下来的URL</span></span><br><span class="line">        <span class="comment"># 需要注意哈！ 不能直接直接改！只能通过Response.replace这个魔术方法来改！（当然你改无所谓啦！反正会用报错来报复你 (`皿´) ）并且！！！</span></span><br><span class="line">        <span class="comment"># 敲黑板！！！！划重点！！！！！注意了！！！ 这货只能赋给一个新的对象（你说变量也行，怎么说都行！(*ﾟ∀ﾟ)=3）</span></span><br><span class="line">        newresponse = response.replace(url=response.meta.get(<span class="string">'real_url'</span>))</span><br><span class="line">        <span class="keyword">for</span> n, rule <span class="keyword">in</span> enumerate(self._rules):</span><br><span class="line">            <span class="comment"># 我要长一点不然有人看不见------------------------------------newresponse 看见没！别忘了改！！！</span></span><br><span class="line">            links = [lnk <span class="keyword">for</span> lnk <span class="keyword">in</span> rule.link_extractor.extract_links(newresponse)</span><br><span class="line">                     <span class="keyword">if</span> lnk <span class="keyword">not</span> <span class="keyword">in</span> seen]</span><br><span class="line">            <span class="keyword">if</span> links <span class="keyword">and</span> rule.process_links:</span><br><span class="line">                links = rule.process_links(links)</span><br><span class="line">            <span class="keyword">for</span> link <span class="keyword">in</span> links:</span><br><span class="line">                seen.add(link)</span><br><span class="line">                r = self._build_request(n, link)</span><br><span class="line">                <span class="keyword">yield</span> rule.process_request(r)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">one_parse</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        print(response.url)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="方法2"><a href="#方法2" class="headerlink" title="方法2:"></a>方法2:</h2>
                  <p>这就很简单啦！干掉类型检查就是了(/≧▽≦)/ 就像这样：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MySpider</span><span class="params">(CrawlSpider)</span>:</span></span><br><span class="line"></span><br><span class="line">    name = <span class="string">'innda'</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">start_requests</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="keyword">yield</span> SplashRequest(url, args=&#123;<span class="string">'wait'</span>: <span class="number">0.5</span>&#125;)</span><br><span class="line"></span><br><span class="line">    rules = (</span><br><span class="line">        Rule(LinkExtractor(allow=(<span class="string">'node_\d+\.htm'</span>,)), process_request=<span class="string">'splash_request'</span>, follow=<span class="literal">True</span>),</span><br><span class="line">        Rule(LinkExtractor(allow=(<span class="string">'content_\d+\.htm'</span>,)), callback=<span class="string">"one_parse"</span>)</span><br><span class="line">    )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">splash_request</span><span class="params">(self, request)</span>:</span></span><br><span class="line">        <span class="string">"""</span></span><br><span class="line"><span class="string">        :param request: Request对象（是一个字典；怎么取值就不说了吧！！）</span></span><br><span class="line"><span class="string">        :return: SplashRequest的请求</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="comment"># dont_process_response=True 参数表示不更改响应对象类型（默认为：HTMLResponse；更改后为：SplashTextResponse）</span></span><br><span class="line">        <span class="comment"># args=&#123;'wait': 0.5&#125; 表示传递等待参数0.5（Splash会渲染0.5s的时间）</span></span><br><span class="line">        <span class="comment"># meta 传递请求的当前请求的URL</span></span><br><span class="line">        <span class="keyword">return</span> SplashRequest(url=request.url, args=&#123;<span class="string">'wait'</span>: <span class="number">0.5</span>&#125;)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_requests_to_follow</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        <span class="string">"""重写的函数哈！这个函数是Rule的一个方法</span></span><br><span class="line"><span class="string">        :param response: 这货是啥看名字都知道了吧（这货也是个字典，然后你懂的ｄ(･∀･*)♪ﾟ）</span></span><br><span class="line"><span class="string">        :return: 追踪的Request</span></span><br><span class="line"><span class="string">        """</span></span><br><span class="line">        <span class="comment"># *************请注意我就是被注释注释掉的类型检查o(TωT)o </span></span><br><span class="line">        <span class="comment"># if not isinstance(response, HtmlResponse):</span></span><br><span class="line">        <span class="comment">#     return</span></span><br><span class="line">        <span class="comment"># ************************************************</span></span><br><span class="line">        seen = set()</span><br><span class="line">        <span class="comment"># 将Response的URL更改为我们传递下来的URL</span></span><br><span class="line">        <span class="comment"># 需要注意哈！ 不能直接直接改！只能通过Response.replace这个魔术方法来改！并且！！！</span></span><br><span class="line">        <span class="comment"># 敲黑板！！！！划重点！！！！！注意了！！！ 这货只能赋给一个新的对象（你说变量也行，怎么说都行！(*ﾟ∀ﾟ)=3）</span></span><br><span class="line">        <span class="comment"># newresponse = response.replace(url=response.meta.get('real_url'))</span></span><br><span class="line">        <span class="keyword">for</span> n, rule <span class="keyword">in</span> enumerate(self._rules):</span><br><span class="line">            <span class="comment"># 我要长一点不然有人看不见------------------------------------newresponse 看见没！别忘了改！！！</span></span><br><span class="line">            links = [lnk <span class="keyword">for</span> lnk <span class="keyword">in</span> rule.link_extractor.extract_links(response)</span><br><span class="line">                     <span class="keyword">if</span> lnk <span class="keyword">not</span> <span class="keyword">in</span> seen]</span><br><span class="line">            <span class="keyword">if</span> links <span class="keyword">and</span> rule.process_links:</span><br><span class="line">                links = rule.process_links(links)</span><br><span class="line">            <span class="keyword">for</span> link <span class="keyword">in</span> links:</span><br><span class="line">                seen.add(link)</span><br><span class="line">                r = self._build_request(n, link)</span><br><span class="line">                <span class="keyword">yield</span> rule.process_request(r)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上完毕@_@!!</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-07-12 00:48:25" itemprop="dateCreated datePublished" datetime="2017-07-12T00:48:25+08:00">2017-07-12</time>
                </span>
                <span id="/4725.html" class="post-meta-item leancloud_visitors" data-flag-title="小白进阶之Scrapy第五篇（Scrapy-Splash配合CrawlSpider；瞎几把整的）" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.4k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4652.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4652.html" class="post-title-link" itemprop="url">利用新接口抓取微信公众号的所有文章</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>各位小伙儿伴儿，一定深受过采集微信公众号之苦吧！特别是！！！！！！公共号历史信息！！！这丫除了通过中间代理采集APP、还真没什么招数能拿到数据啊！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" alt=""></a> 直到············ 前天晚上微信官方发布了一个文章：<a href="http://mp.weixin.qq.com/s/67sk-uKz9Ct4niT-f4u1KA" target="_blank" rel="noopener">点我</a> 大致意思是说以后发布文章的时候可以直接插入其它公众号的文章了。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224219.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161021224219.gif" alt=""></a> 诶妈呀！这不是一直需要的采集接口嘛！啧啧 天助我也啊！来来·········下面大致的说一下方法。</p>
                  <h2 id="1、首先你需要一个订阅号！-公众号、和企业号是否可行我不清楚。因为我木有·····"><a href="#1、首先你需要一个订阅号！-公众号、和企业号是否可行我不清楚。因为我木有·····" class="headerlink" title="1、首先你需要一个订阅号！ 公众号、和企业号是否可行我不清楚。因为我木有·····"></a>1、首先你需要一个订阅号！ 公众号、和企业号是否可行我不清楚。因为我木有·····</h2>
                  <h2 id="2、其次你需要登录！"><a href="#2、其次你需要登录！" class="headerlink" title="2、其次你需要登录！"></a><strong>2、其次你需要登录！</strong></h2>
                  <p>微信公众号登录我没仔细看。 这个暂且不说了，我使用的是selenium 驱动浏览器获取Cookie的方法、来达到登录的效果。</p>
                  <h2 id="3、使用requests携带Cookie、登录获取URL的token（这玩意儿很重要每一次请求都需要带上它）像下面这样："><a href="#3、使用requests携带Cookie、登录获取URL的token（这玩意儿很重要每一次请求都需要带上它）像下面这样：" class="headerlink" title="3、使用requests携带Cookie、登录获取URL的token（这玩意儿很重要每一次请求都需要带上它）像下面这样："></a>3、使用requests携带Cookie、登录获取URL的token（这玩意儿很重要每一次请求都需要带上它）像下面这样：</h2>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/TIM截图20170607085814.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/TIM截图20170607085814.png" alt=""></a></p>
                  <h2 id="4、使用获取到的token、和公众号的微信号（就是数字-字符那种）、获取到公众号的fakeid（你可以理解公众号的标识）"><a href="#4、使用获取到的token、和公众号的微信号（就是数字-字符那种）、获取到公众号的fakeid（你可以理解公众号的标识）" class="headerlink" title="4、使用获取到的token、和公众号的微信号（就是数字+字符那种）、获取到公众号的fakeid（你可以理解公众号的标识）"></a>4、使用获取到的token、和公众号的微信号（就是数字+字符那种）、获取到公众号的fakeid（你可以理解公众号的标识）</h2>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/2.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/2.png" alt=""></a></p>
                  <h3 id="我们在搜索公众号的时候浏览器带着参数以GET方法想红框中的URL发起了请求。请求参数如下："><a href="#我们在搜索公众号的时候浏览器带着参数以GET方法想红框中的URL发起了请求。请求参数如下：" class="headerlink" title="我们在搜索公众号的时候浏览器带着参数以GET方法想红框中的URL发起了请求。请求参数如下："></a>我们在搜索公众号的时候浏览器带着参数以GET方法想红框中的URL发起了请求。请求参数如下：</h3>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/3.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/3.png" alt=""></a></p>
                  <h3 id="请求相应如下："><a href="#请求相应如下：" class="headerlink" title="请求相应如下："></a><strong>请求相应如下：</strong></h3>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/4.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/4.png" alt=""></a></p>
                  <h3 id="代码如下："><a href="#代码如下：" class="headerlink" title="代码如下："></a><strong>代码如下：</strong></h3>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/5.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/5.png" alt=""></a></p>
                  <h1 id="好了-我们再继续："><a href="#好了-我们再继续：" class="headerlink" title="好了 我们再继续："></a>好了 我们再继续：</h1>
                  <h2 id="5、点击我们搜索到的公众号之后、又发现一个请求："><a href="#5、点击我们搜索到的公众号之后、又发现一个请求：" class="headerlink" title="5、点击我们搜索到的公众号之后、又发现一个请求："></a><strong>5、点击我们搜索到的公众号之后、又发现一个请求：</strong></h2>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/6.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/6.png" alt=""></a></p>
                  <h3 id="请求参数如下："><a href="#请求参数如下：" class="headerlink" title="请求参数如下："></a>请求参数如下：</h3>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/7.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/7.png" alt=""></a></p>
                  <h3 id="返回如下："><a href="#返回如下：" class="headerlink" title="返回如下："></a>返回如下：</h3>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/8.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/8.png" alt=""></a></p>
                  <h3 id="代码如下：-1"><a href="#代码如下：-1" class="headerlink" title="代码如下："></a>代码如下：</h3>
                  <p> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/9.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/9.png" alt=""></a></p>
                  <h2 id="好了···最后一步、获取所有文章需要处理一下翻页、翻页请求如下："><a href="#好了···最后一步、获取所有文章需要处理一下翻页、翻页请求如下：" class="headerlink" title="好了···最后一步、获取所有文章需要处理一下翻页、翻页请求如下："></a>好了···最后一步、获取所有文章需要处理一下翻页、翻页请求如下：</h2>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/10.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/10.png" alt=""></a></p>
                  <h4 id="我大概看了一下、极客学院每一页大概至少有5条信息、也就是总文章数-5-就是有多少页。但是有小数、我们取整，然后加1就是总页数了。"><a href="#我大概看了一下、极客学院每一页大概至少有5条信息、也就是总文章数-5-就是有多少页。但是有小数、我们取整，然后加1就是总页数了。" class="headerlink" title="我大概看了一下、极客学院每一页大概至少有5条信息、也就是总文章数/5 就是有多少页。但是有小数、我们取整，然后加1就是总页数了。"></a><strong>我大概看了一下、极客学院每一页大概至少有5条信息、也就是总文章数/5 就是有多少页。但是有小数、我们取整，然后加1就是总页数了。</strong></h4>
                  <h4 id="代码如下：-2"><a href="#代码如下：-2" class="headerlink" title="代码如下："></a><strong>代码如下：</strong></h4>
                  <p> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/11.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/11.png" alt=""></a></p>
                  <h2 id="item-get-‘link’-就是我们需要的公众号文章连接啦！继续请求这个URL提取里面的内容就是啦！"><a href="#item-get-‘link’-就是我们需要的公众号文章连接啦！继续请求这个URL提取里面的内容就是啦！" class="headerlink" title="item.get(‘link’)就是我们需要的公众号文章连接啦！继续请求这个URL提取里面的内容就是啦！"></a><strong>item.get(‘link’)就是我们需要的公众号文章连接啦！继续请求这个URL提取里面的内容就是啦！</strong></h2>
                  <h1 id="以下是完整的测试代码："><a href="#以下是完整的测试代码：" class="headerlink" title="以下是完整的测试代码："></a><strong>以下是完整的测试代码：</strong></h1>
                  <figure class="highlight xl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">import</span> <span class="built_in">time</span></span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line">from pprint <span class="keyword">import</span> pprint</span><br><span class="line"></span><br><span class="line">post = &#123;&#125;</span><br><span class="line"></span><br><span class="line">driver = webdriver.Chrome(executable_path=<span class="string">'C:\chromedriver.exe'</span>)</span><br><span class="line">driver.get(<span class="string">'https://mp.weixin.qq.com/'</span>)</span><br><span class="line"><span class="built_in">time</span>.sleep(<span class="number">2</span>)</span><br><span class="line">driver.find_element_by_xpath(<span class="string">"./*//input[@id='account']"</span>).clear()</span><br><span class="line">driver.find_element_by_xpath(<span class="string">"./*//input[@id='account']"</span>).send_keys(<span class="string">'你的帐号'</span>)</span><br><span class="line">driver.find_element_by_xpath(<span class="string">"./*//input[@id='pwd']"</span>).clear()</span><br><span class="line">driver.find_element_by_xpath(<span class="string">"./*//input[@id='pwd']"</span>).send_keys(<span class="string">'你的密码'</span>)</span><br><span class="line"># 在自动输完密码之后记得点一下记住我</span><br><span class="line"><span class="built_in">time</span>.sleep(<span class="number">5</span>)</span><br><span class="line">driver.find_element_by_xpath(<span class="string">"./*//a[@id='loginBt']"</span>).click()</span><br><span class="line"># 拿手机扫二维码！</span><br><span class="line"><span class="built_in">time</span>.sleep(<span class="number">15</span>)</span><br><span class="line">driver.get(<span class="string">'https://mp.weixin.qq.com/'</span>)</span><br><span class="line">cookie_items = driver.get_cookies()</span><br><span class="line"><span class="keyword">for</span> cookie_item <span class="built_in">in</span> cookie_items:</span><br><span class="line">    post[cookie_item[<span class="string">'name'</span>]] = cookie_item[<span class="string">'value'</span>]</span><br><span class="line">cookie_str = json.dumps(post)</span><br><span class="line"><span class="keyword">with</span> open(<span class="string">'cookie.txt'</span>, <span class="string">'w+'</span>, encoding=<span class="string">'utf-8'</span>) <span class="keyword">as</span> f:</span><br><span class="line">    f.write(cookie_str)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line">import redis</span><br><span class="line">import json</span><br><span class="line">import re</span><br><span class="line">import random</span><br><span class="line">import time</span><br><span class="line"></span><br><span class="line">gzlist = [<span class="string">'yq_Butler'</span>]</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">url = <span class="string">'https://mp.weixin.qq.com'</span></span><br><span class="line">header = &#123;</span><br><span class="line">    <span class="string">"HOST"</span>: <span class="string">"mp.weixin.qq.com"</span>,</span><br><span class="line">    <span class="string">"User-Agent"</span>: <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0"</span></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">with open(<span class="string">'cookie.txt'</span>, <span class="string">'r'</span>, <span class="attribute">encoding</span>=<span class="string">'utf-8'</span>) as f:</span><br><span class="line">    cookie = f.read()</span><br><span class="line">cookies = json.loads(cookie)</span><br><span class="line">response = requests.<span class="builtin-name">get</span>(<span class="attribute">url</span>=url, <span class="attribute">cookies</span>=cookies)</span><br><span class="line">token = re.findall(r<span class="string">'token=(\d+)'</span>, str(response.url))[0]</span><br><span class="line"><span class="keyword">for</span> query <span class="keyword">in</span> gzlist:</span><br><span class="line">    query_id = &#123;</span><br><span class="line">        <span class="string">'action'</span>: <span class="string">'search_biz'</span>,</span><br><span class="line">        <span class="string">'token'</span> : token,</span><br><span class="line">        <span class="string">'lang'</span>: <span class="string">'zh_CN'</span>,</span><br><span class="line">        <span class="string">'f'</span>: <span class="string">'json'</span>,</span><br><span class="line">        <span class="string">'ajax'</span>: <span class="string">'1'</span>,</span><br><span class="line">        <span class="string">'random'</span>: random.random(),</span><br><span class="line">        <span class="string">'query'</span>: query,</span><br><span class="line">        <span class="string">'begin'</span>: <span class="string">'0'</span>,</span><br><span class="line">        <span class="string">'count'</span>: <span class="string">'5'</span>,</span><br><span class="line">    &#125;</span><br><span class="line">    search_url = <span class="string">'https://mp.weixin.qq.com/cgi-bin/searchbiz?'</span></span><br><span class="line">    search_response = requests.<span class="builtin-name">get</span>(search_url, <span class="attribute">cookies</span>=cookies, <span class="attribute">headers</span>=header, <span class="attribute">params</span>=query_id)</span><br><span class="line">    lists = search_response.json().<span class="builtin-name">get</span>(<span class="string">'list'</span>)[0]</span><br><span class="line">    fakeid = lists.<span class="builtin-name">get</span>(<span class="string">'fakeid'</span>)</span><br><span class="line">    query_id_data = &#123;</span><br><span class="line">        <span class="string">'token'</span>: token,</span><br><span class="line">        <span class="string">'lang'</span>: <span class="string">'zh_CN'</span>,</span><br><span class="line">        <span class="string">'f'</span>: <span class="string">'json'</span>,</span><br><span class="line">        <span class="string">'ajax'</span>: <span class="string">'1'</span>,</span><br><span class="line">        <span class="string">'random'</span>: random.random(),</span><br><span class="line">        <span class="string">'action'</span>: <span class="string">'list_ex'</span>,</span><br><span class="line">        <span class="string">'begin'</span>: <span class="string">'0'</span>,</span><br><span class="line">        <span class="string">'count'</span>: <span class="string">'5'</span>,</span><br><span class="line">        <span class="string">'query'</span>: <span class="string">''</span>,</span><br><span class="line">        <span class="string">'fakeid'</span>: fakeid,</span><br><span class="line">        <span class="string">'type'</span>: <span class="string">'9'</span></span><br><span class="line">    &#125;</span><br><span class="line">    appmsg_url = <span class="string">'https://mp.weixin.qq.com/cgi-bin/appmsg?'</span></span><br><span class="line">    appmsg_response = requests.<span class="builtin-name">get</span>(appmsg_url, <span class="attribute">cookies</span>=cookies, <span class="attribute">headers</span>=header, <span class="attribute">params</span>=query_id_data)</span><br><span class="line">    max_num = appmsg_response.json().<span class="builtin-name">get</span>(<span class="string">'app_msg_cnt'</span>)</span><br><span class="line">    num = int(int(max_num) / 5)</span><br><span class="line">    begin = 0</span><br><span class="line">    <span class="keyword">while</span> num + 1 &gt; 0 :</span><br><span class="line">        query_id_data = &#123;</span><br><span class="line">            <span class="string">'token'</span>: token,</span><br><span class="line">            <span class="string">'lang'</span>: <span class="string">'zh_CN'</span>,</span><br><span class="line">            <span class="string">'f'</span>: <span class="string">'json'</span>,</span><br><span class="line">            <span class="string">'ajax'</span>: <span class="string">'1'</span>,</span><br><span class="line">            <span class="string">'random'</span>: random.random(),</span><br><span class="line">            <span class="string">'action'</span>: <span class="string">'list_ex'</span>,</span><br><span class="line">            <span class="string">'begin'</span>: <span class="string">'&#123;&#125;'</span>.format(str(begin)),</span><br><span class="line">            <span class="string">'count'</span>: <span class="string">'5'</span>,</span><br><span class="line">            <span class="string">'query'</span>: <span class="string">''</span>,</span><br><span class="line">            <span class="string">'fakeid'</span>: fakeid,</span><br><span class="line">            <span class="string">'type'</span>: <span class="string">'9'</span></span><br><span class="line">        &#125;</span><br><span class="line">        <span class="builtin-name">print</span>(<span class="string">'翻页###################'</span>,begin)</span><br><span class="line">        query_fakeid_response = requests.<span class="builtin-name">get</span>(appmsg_url, <span class="attribute">cookies</span>=cookies, <span class="attribute">headers</span>=header, <span class="attribute">params</span>=query_id_data)</span><br><span class="line">        fakeid_list = query_fakeid_response.json().<span class="builtin-name">get</span>(<span class="string">'app_msg_list'</span>)</span><br><span class="line">        <span class="keyword">for</span> item <span class="keyword">in</span> fakeid_list:</span><br><span class="line">            <span class="builtin-name">print</span>(item.<span class="builtin-name">get</span>(<span class="string">'link'</span>))</span><br><span class="line">        num -= 1</span><br><span class="line">        begin = int(begin)</span><br><span class="line">        begin+=5</span><br><span class="line">        time.sleep(2)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h1 id="以上完毕！这就是个测试、代码写得奇丑、各位将就着看啊！看不明白？没关系！看这儿：点我看视频"><a href="#以上完毕！这就是个测试、代码写得奇丑、各位将就着看啊！看不明白？没关系！看这儿：点我看视频" class="headerlink" title="以上完毕！这就是个测试、代码写得奇丑、各位将就着看啊！看不明白？没关系！看这儿：点我看视频"></a><strong>以上完毕！这就是个测试、代码写得奇丑、各位将就着看啊！看不明白？没关系！看这儿：<a href="http://www.bilibili.com/video/av11127609/" target="_blank" rel="noopener">点我看视频</a></strong></h1>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-06-07 09:44:51" itemprop="dateCreated datePublished" datetime="2017-06-07T09:44:51+08:00">2017-06-07</time>
                </span>
                <span id="/4652.html" class="post-meta-item leancloud_visitors" data-flag-title="利用新接口抓取微信公众号的所有文章" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4607.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4607.html" class="post-title-link" itemprop="url">获取知乎问题答案并转换为MarkDown文件</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <blockquote>
                    <h2 id="20170609-更新"><a href="#20170609-更新" class="headerlink" title="20170609 更新:"></a>20170609 更新:</h2>
                    <p><strong>感谢一介草民与ftzz的反馈</strong></p>
                    <h3 id="1-修复中文路径保存问题"><a href="#1-修复中文路径保存问题" class="headerlink" title="(1) 修复中文路径保存问题"></a>(1) 修复中文路径保存问题</h3>
                    <h3 id="2-修复offset问题"><a href="#2-修复offset问题" class="headerlink" title="(2) 修复offset问题"></a>(2) 修复offset问题</h3>
                    <h3 id="3-修复第一个问题"><a href="#3-修复第一个问题" class="headerlink" title="(3) 修复第一个问题"></a>(3) 修复第一个问题</h3>
                    <h2 id="来个好玩的东西"><a href="#来个好玩的东西" class="headerlink" title="来个好玩的东西"></a>来个好玩的东西</h2>
                    <h2 id=""><a href="#" class="headerlink" title="     "></a><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/6f19c6ad326822c9e267d2d961cf1fec_r.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/6f19c6ad326822c9e267d2d961cf1fec_r.png" alt=""></a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/154e026013e7ec53e8ce94c8b4417973_r.jpeg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/154e026013e7ec53e8ce94c8b4417973_r.jpeg" alt=""></a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/0838fb7d2c9d61070605148ab57f90cb_r.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/0838fb7d2c9d61070605148ab57f90cb_r.png" alt=""></a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/a1e43e58c01f1e36630f4a1394811b67_r.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/a1e43e58c01f1e36630f4a1394811b67_r.png" alt=""></a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/d851605dddb3e14ad9946a3eccc0ae05_r.jpeg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/d851605dddb3e14ad9946a3eccc0ae05_r.jpeg" alt=""></a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/f8acc0c27d15fcfd8b2c9682aabe6633_r.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/f8acc0c27d15fcfd8b2c9682aabe6633_r.png" alt=""></a></h2>
                    <h2 id="20170607-更新"><a href="#20170607-更新" class="headerlink" title="20170607 更新:"></a>20170607 更新:</h2>
                    <h3 id="1-感谢Ftzz提醒-将图片替换为原图"><a href="#1-感谢Ftzz提醒-将图片替换为原图" class="headerlink" title="(1) 感谢Ftzz提醒, 将图片替换为原图"></a>(1) 感谢Ftzz提醒, 将图片替换为原图</h3>
                    <h3 id="2-将文件保存到本地-解决了最大的缺点问题-不用联网也可以看了"><a href="#2-将文件保存到本地-解决了最大的缺点问题-不用联网也可以看了" class="headerlink" title="(2) 将文件保存到本地,解决了最大的缺点问题,不用联网也可以看了"></a>(2) 将文件保存到本地,解决了最大的缺点问题,不用联网也可以看了</h3>
                  </blockquote>
                  <p> 大家好，我是四毛。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/02/QQ图片20170205084843.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/02/QQ图片20170205084843.jpg" alt=""></a> <strong>写在前面的话</strong> 在开始前，给大家分享一个前段时间逛Github时看到的某个爬虫脚本中的内容： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/lvshi.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/lvshi.jpg" alt=""></a> 所以，大家爬网站的时候，还是友善一点为好，且爬且珍惜啊。 好了，言归正传。 今天主要讲一下如何将某一个知乎问题的所有答案转换为本地MarkDown文件。</p>
                  <h2 id="前期准备"><a href="#前期准备" class="headerlink" title="前期准备"></a>前期准备</h2>
                  <blockquote>
                    <p>python2.7 html2text markdownpad(这里随意，只要可以支持md就行) 会抓包。。。。。 最重要的是你要有代理，因为知乎开始封IP了</p>
                  </blockquote>
                  <h2 id="1-什么是MarkDown文件"><a href="#1-什么是MarkDown文件" class="headerlink" title="1.什么是MarkDown文件"></a><strong>1.什么是MarkDown文件</strong></h2>
                  <p>Markdown 是一种用来写作的轻量级<strong>「标记语言」</strong>，它用简洁的语法代替排版，而不像一般我们用的字处理软件 <em>Word</em> 或 <em>Pages</em> 有大量的排版、字体设置。它使我们专心于码字，用「标记」语法，来代替常见的排版格式。例如此文从内容到格式，甚至插图，键盘就可以通通搞定了。 恩，上面是我抄的，哈哈。想多了解的可以看看<a href="http://www.jianshu.com/p/1e402922ee32/" target="_blank" rel="noopener">这里</a>。</p>
                  <h2 id="2-为什么要将答案转为MarkDwon"><a href="#2-为什么要将答案转为MarkDwon" class="headerlink" title="2.为什么要将答案转为MarkDwon"></a><strong>2.为什么要将答案转为MarkDwon</strong></h2>
                  <p>因为。。。。。。懒，哈哈，开个玩笑。最重要的原因还是markdown看着比较舒服。平时写脚本的时候，也一直在思考一个问题，如何将一个文字与图片穿插的网页原始的保存下来呢。如果借助工具的话，那就很多了，CTRL+P 打印的时候，选择另存为PDF，或者搞个印象笔记，直接保存整个网页。那么，我们如何用爬虫实现呢？正好前几天看到了<a href="https://github.com/egrcc/zhihu-python" target="_blank" rel="noopener">这个项目</a>，仔细研究了一下，大受启发。</p>
                  <h2 id="3-原理"><a href="#3-原理" class="headerlink" title="3.原理"></a><strong>3.原理</strong></h2>
                  <p>原理说起来很简单：获取请求到的内容的BODY部分，然后重新构建一个HTML文件，接着利用html2text这个模块将其转换为markdown文件，最后对图片及标题按照markdown的格式做一些处理就好了。目前应用的场景主要是在知乎。</p>
                  <h2 id="4-Show-Code"><a href="#4-Show-Code" class="headerlink" title="4.Show Code"></a><strong>4.Show Code</strong></h2>
                  <h3 id="4-1获取知乎答案"><a href="#4-1获取知乎答案" class="headerlink" title="4.1获取知乎答案"></a>4.1获取知乎答案</h3>
                  <p>写代码的时候，主要考虑了两种使用场景。第一，获取某一特定答案的数据然后进行转换；第二，获取某一个问题的所有答案进行然后挨个进行转换，在这里可以 通过赞同数来对要获取的答案进行质量控制。 <strong> 4.1.1、某一个特定答案的数据获取</strong></p>
                  <blockquote>
                    <p>url：<a href="https://www.zhihu.com/question/27621722/answer/48658220（前面那个是问题ID，后边的是答案ID）" target="_blank" rel="noopener">https://www.zhihu.com/question/27621722/answer/48658220（前面那个是问题ID，后边的是答案ID）</a></p>
                  </blockquote>
                  <p>这一数据的获取我这里分为了两个部分，第一部分请求上述网址，拿到答案主体数据以及赞同数，第二部分请求下面这个接口：</p>
                  <blockquote>
                    <p><a href="https://www.zhihu.com/api/v4/answers/48658220" target="_blank" rel="noopener">https://www.zhihu.com/api/v4/answers/48658220</a></p>
                  </blockquote>
                  <p>为什么会这样？因为这个接口得到的答案正文数据不是完整数据，所以只能分两步了。 <strong> 4.1.2、某一个特定答案的数据获取</strong> 这一个数据就可以通过很简单的方式得到了，接口如下：</p>
                  <blockquote>
                    <p><a href="https://www.zhihu.com/api/v4/questions/27621722/answers?sort_by=default&amp;include=data%5B%2A%5D.is_normal%2Cis_collapsed%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Cmark_infos%2Ccreated_time%2Cupdated_time%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.author.follower_count%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics&amp;limit=20&amp;offset=3" target="_blank" rel="noopener">https://www.zhihu.com/api/v4/questions/27621722/answers?sort_by=default&amp;include=data%5B%2A%5D.is_normal%2Cis_collapsed%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Cmark_infos%2Ccreated_time%2Cupdated_time%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.author.follower_count%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics&amp;limit=20&amp;offset=3</a></p>
                  </blockquote>
                  <p>返回的都是JSON数据，很方便获取。但是这里有一个地方需要注意，从这里面取的答案正文数据就是文本数据，不是一个完整的html文件，所以需要在构造一下。 <strong> 4.1.2、保存的字段</strong></p>
                  <blockquote>
                    <p>author_name 回答用户名 answer_id 答案ID question_id 问题ID question_title 问题 vote_up_count 赞同数 create_time 创建时间 答案主体 </p>
                  </blockquote>
                  <h3 id="4-2-Code"><a href="#4-2-Code" class="headerlink" title="4.2 Code"></a>4.2 Code</h3>
                  <p>主脚本：zhihu.py</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">#!/usr/bin/env python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># Created by shimeng on 17-6-5</span></span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> html2text</span><br><span class="line"><span class="keyword">from</span> parse_content <span class="keyword">import</span> parse</span><br><span class="line"></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">just for study and fun</span></span><br><span class="line"><span class="string">Talk is cheap</span></span><br><span class="line"><span class="string">show me your code</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ZhiHu</span><span class="params">(object)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line">         self.request_content = <span class="literal">None</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">request</span><span class="params">(self, url, retry_times=<span class="number">10</span>)</span>:</span></span><br><span class="line">        header = &#123;</span><br><span class="line">            <span class="string">'User-Agent'</span>: <span class="string">'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'</span>,</span><br><span class="line">            <span class="string">'authorization'</span>: <span class="string">'oauth c3cef7c66a1843f8b3a9e6a1e3160e20'</span>,</span><br><span class="line">            <span class="string">'Host'</span>: <span class="string">'www.zhihu.com'</span></span><br><span class="line">        &#125;</span><br><span class="line">        times = <span class="number">0</span></span><br><span class="line">        <span class="keyword">while</span> retry_times&gt;<span class="number">0</span>:</span><br><span class="line">            times += <span class="number">1</span></span><br><span class="line">            <span class="keyword">print</span> <span class="string">'request %s, times: %d'</span> %(url, times)</span><br><span class="line">            <span class="keyword">try</span>:</span><br><span class="line">                ip = <span class="string">'your proxy ip'</span></span><br><span class="line">                <span class="keyword">if</span> ip:</span><br><span class="line">                    proxy = &#123;</span><br><span class="line">                        <span class="string">'http'</span>: <span class="string">'http://%s'</span> % ip,</span><br><span class="line">                        <span class="string">'https'</span>: <span class="string">'http://%s'</span> % ip</span><br><span class="line">                    &#125;</span><br><span class="line">                    self.request_content = requests.get(url, headers=header, proxies=proxy, timeout=<span class="number">10</span>).content</span><br><span class="line">            <span class="keyword">except</span> Exception, e:</span><br><span class="line">                <span class="keyword">print</span> e</span><br><span class="line">                retry_times -= <span class="number">1</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                <span class="keyword">return</span> self.request_content</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get_all_answer_content</span><span class="params">(self, question_id, flag=<span class="number">2</span>)</span>:</span></span><br><span class="line">        first_url_format = <span class="string">'https://www.zhihu.com/api/v4/questions/&#123;&#125;/answers?sort_by=default&amp;include=data%5B%2A%5D.is_normal%2Cis_collapsed%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Cmark_infos%2Ccreated_time%2Cupdated_time%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.author.follower_count%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics&amp;limit=20&amp;offset=3'</span></span><br><span class="line">        first_url = first_url_format.format(question_id)</span><br><span class="line">        response = self.request(first_url)</span><br><span class="line">        <span class="keyword">if</span> response:</span><br><span class="line">            contents = json.loads(response)</span><br><span class="line">            <span class="keyword">print</span> contents.get(<span class="string">'paging'</span>).get(<span class="string">'is_end'</span>)</span><br><span class="line">            <span class="keyword">while</span> <span class="keyword">not</span> contents.get(<span class="string">'paging'</span>).get(<span class="string">'is_end'</span>):</span><br><span class="line">                <span class="keyword">for</span> content <span class="keyword">in</span> contents.get(<span class="string">'data'</span>):</span><br><span class="line">                    self.parse_content(content, flag)</span><br><span class="line">                next_page_url = contents.get(<span class="string">'paging'</span>).get(<span class="string">'next'</span>).replace(<span class="string">'http'</span>, <span class="string">'https'</span>)</span><br><span class="line">                contents = json.loads(self.request(next_page_url))</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">raise</span> ValueError(<span class="string">'request failed, quit......'</span>)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get_single_answer_content</span><span class="params">(self, answer_url, flag=<span class="number">1</span>)</span>:</span></span><br><span class="line">        all_content = &#123;&#125;</span><br><span class="line">        question_id, answer_id = re.findall(<span class="string">'https://www.zhihu.com/question/(\d+)/answer/(\d+)'</span>, answer_url)[<span class="number">0</span>]</span><br><span class="line"></span><br><span class="line">        html_content = self.request(answer_url)</span><br><span class="line">        <span class="keyword">if</span> html_content:</span><br><span class="line">            all_content[<span class="string">'main_content'</span>] = html_content</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">raise</span>  ValueError(<span class="string">'request failed, quit......'</span>)</span><br><span class="line"></span><br><span class="line">        ajax_answer_url = <span class="string">'https://www.zhihu.com/api/v4/answers/&#123;&#125;'</span>.format(answer_id)</span><br><span class="line">        ajax_content = self.request(ajax_answer_url)</span><br><span class="line">        <span class="keyword">if</span> ajax_content:</span><br><span class="line">            all_content[<span class="string">'ajax_content'</span>] = json.loads(ajax_content)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">raise</span>  ValueError(<span class="string">'request failed, quit......'</span>)</span><br><span class="line"></span><br><span class="line">        self.parse_content(all_content, flag, )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse_content</span><span class="params">(self, content, flag=None)</span>:</span></span><br><span class="line">        data = parse(content, flag)</span><br><span class="line">        self.transform_to_markdown(data)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">transform_to_markdown</span><span class="params">(self, data)</span>:</span></span><br><span class="line">        content = data[<span class="string">'content'</span>]</span><br><span class="line">        author_name = data[<span class="string">'author_name'</span>]</span><br><span class="line">        answer_id = data[<span class="string">'answer_id'</span>]</span><br><span class="line">        question_id = data[<span class="string">'question_id'</span>]</span><br><span class="line">        question_title = data[<span class="string">'question_title'</span>]</span><br><span class="line">        vote_up_count = data[<span class="string">'vote_up_count'</span>]</span><br><span class="line">        create_time = data[<span class="string">'create_time'</span>]</span><br><span class="line"></span><br><span class="line">        file_name = <span class="string">u'%s--%s的回答[%d].md'</span> % (question_title, author_name,answer_id)</span><br><span class="line">        folder_name = <span class="string">u'%s'</span> % (question_title)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(os.path.join(os.getcwd(),folder_name)):</span><br><span class="line">            os.mkdir(folder_name)</span><br><span class="line">        os.chdir(folder_name)</span><br><span class="line"></span><br><span class="line">        f = open(file_name, <span class="string">"wt"</span>)</span><br><span class="line">        f.write(<span class="string">"-"</span> * <span class="number">40</span> + <span class="string">"\n"</span>)</span><br><span class="line">        origin_url = <span class="string">'https://www.zhihu.com/question/&#123;&#125;/answer/&#123;&#125;'</span>.format(question_id, answer_id)</span><br><span class="line">        f.write(<span class="string">"## 本答案原始链接: "</span> + origin_url + <span class="string">"\n"</span>)</span><br><span class="line">        f.write(<span class="string">"### question_title: "</span> + question_title.encode(<span class="string">'utf-8'</span>) + <span class="string">"\n"</span>)</span><br><span class="line">        f.write(<span class="string">"### Author_Name: "</span> + author_name.encode(<span class="string">'utf-8'</span>) + <span class="string">"\n"</span>)</span><br><span class="line">        f.write(<span class="string">"### Answer_ID: %d"</span> % answer_id + <span class="string">"\n"</span>)</span><br><span class="line">        f.write(<span class="string">"### Question_ID %d: "</span> % question_id + <span class="string">"\n"</span>)</span><br><span class="line">        f.write(<span class="string">"### VoteCount: %s"</span> % vote_up_count + <span class="string">"\n"</span>)</span><br><span class="line">        f.write(<span class="string">"### Create_Time: "</span> + create_time + <span class="string">"\n"</span>)</span><br><span class="line">        f.write(<span class="string">"-"</span> * <span class="number">40</span> + <span class="string">"\n"</span>)</span><br><span class="line"></span><br><span class="line">        text = html2text.html2text(content.decode(<span class="string">'utf-8'</span>)).encode(<span class="string">"utf-8"</span>)</span><br><span class="line">        <span class="comment"># 标题</span></span><br><span class="line">        r = re.findall(<span class="string">r'**(.*?)**'</span>, text, re.S)</span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> r:</span><br><span class="line">            <span class="keyword">if</span> i != <span class="string">" "</span>:</span><br><span class="line">                text = text.replace(i, i.strip())</span><br><span class="line"></span><br><span class="line">        r = re.findall(<span class="string">r'_(.*)_'</span>, text)</span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> r:</span><br><span class="line">            <span class="keyword">if</span> i != <span class="string">" "</span>:</span><br><span class="line">                text = text.replace(i, i.strip())</span><br><span class="line">        text = text.replace(<span class="string">'_ _'</span>, <span class="string">''</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 图片</span></span><br><span class="line">        r = re.findall(<span class="string">r'![]\((?:.*?)\)'</span>, text)</span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> r:</span><br><span class="line">            text = text.replace(i, i + <span class="string">"\n\n"</span>)</span><br><span class="line"></span><br><span class="line">        f.write(text)</span><br><span class="line"></span><br><span class="line">        f.close()</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    zhihu = ZhiHu()</span><br><span class="line">    url = <span class="string">'https://www.zhihu.com/question/27621722/answer/105331078'</span></span><br><span class="line">    zhihu.get_single_answer_content(url)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># question_id = '27621722'</span></span><br><span class="line">    <span class="comment"># zhihu.get_all_answer_content(question_id)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>zhihu.py为主脚本，内容很简单，发起请求，调用解析函数进行解析，最后再进行保存。 解析函数脚本：parse_content.py</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">#!/usr/bin/env python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># Created by shimeng on 17-6-5</span></span><br><span class="line">import time</span><br><span class="line"><span class="keyword">from</span> bs4 import BeautifulSoup</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">def html_template(data):</span><br><span class="line">    # api content</span><br><span class="line">    html = <span class="string">''</span><span class="string">'</span></span><br><span class="line"><span class="string">        &lt;html&gt;</span></span><br><span class="line"><span class="string">        &lt;head&gt;</span></span><br><span class="line"><span class="string">        &lt;body&gt;</span></span><br><span class="line"><span class="string">        %s</span></span><br><span class="line"><span class="string">        &lt;/body&gt;</span></span><br><span class="line"><span class="string">        &lt;/head&gt;</span></span><br><span class="line"><span class="string">        &lt;/html&gt;</span></span><br><span class="line"><span class="string">        '</span><span class="string">''</span> % data</span><br><span class="line">    return html</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">def parse(content, <span class="attribute">flag</span>=None):</span><br><span class="line">    data = &#123;&#125;</span><br><span class="line">    <span class="keyword">if</span> flag == 1:</span><br><span class="line">        # single</span><br><span class="line">        main_content = content.<span class="builtin-name">get</span>(<span class="string">'main_content'</span>)</span><br><span class="line">        ajax_content = content.<span class="builtin-name">get</span>(<span class="string">'ajax_content'</span>)</span><br><span class="line"></span><br><span class="line">        soup = BeautifulSoup(main_content.decode(<span class="string">"utf-8"</span>), <span class="string">"lxml"</span>)</span><br><span class="line">        answer = soup.<span class="builtin-name">find</span>(<span class="string">"span"</span>, <span class="attribute">class_</span>=<span class="string">"RichText CopyrightRichText-richText"</span>)</span><br><span class="line"></span><br><span class="line">        author_name = ajax_content.<span class="builtin-name">get</span>(<span class="string">'author'</span>).<span class="builtin-name">get</span>(<span class="string">'name'</span>)</span><br><span class="line">        answer_id = ajax_content.<span class="builtin-name">get</span>(<span class="string">'id'</span>)</span><br><span class="line">        question_id = ajax_content.<span class="builtin-name">get</span>(<span class="string">'question'</span>).<span class="builtin-name">get</span>(<span class="string">'id'</span>)</span><br><span class="line">        question_title = ajax_content.<span class="builtin-name">get</span>(<span class="string">'question'</span>).<span class="builtin-name">get</span>(<span class="string">'title'</span>)</span><br><span class="line">        vote_up_count = soup.<span class="builtin-name">find</span>(<span class="string">"meta"</span>, <span class="attribute">itemprop</span>=<span class="string">"upvoteCount"</span>)[<span class="string">"content"</span>]</span><br><span class="line">        create_time = time.strftime(<span class="string">"%Y-%m-%d %H:%M:%S"</span>, time.localtime(ajax_content.<span class="builtin-name">get</span>(<span class="string">'created_time'</span>)))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        # all</span><br><span class="line">        answer_content = content.<span class="builtin-name">get</span>(<span class="string">'content'</span>)</span><br><span class="line"></span><br><span class="line">        author_name = content.<span class="builtin-name">get</span>(<span class="string">'author'</span>).<span class="builtin-name">get</span>(<span class="string">'name'</span>)</span><br><span class="line">        answer_id = content.<span class="builtin-name">get</span>(<span class="string">'id'</span>)</span><br><span class="line">        question_id = content.<span class="builtin-name">get</span>(<span class="string">'question'</span>).<span class="builtin-name">get</span>(<span class="string">'id'</span>)</span><br><span class="line">        question_title = content.<span class="builtin-name">get</span>(<span class="string">'question'</span>).<span class="builtin-name">get</span>(<span class="string">'title'</span>)</span><br><span class="line">        vote_up_count = content.<span class="builtin-name">get</span>(<span class="string">'voteup_count'</span>)</span><br><span class="line">        create_time = time.strftime(<span class="string">"%Y-%m-%d %H:%M:%S"</span>, time.localtime(content.<span class="builtin-name">get</span>(<span class="string">'created_time'</span>)))</span><br><span class="line"></span><br><span class="line">        content = html_template(answer_content)</span><br><span class="line">        soup = BeautifulSoup(content, <span class="string">'lxml'</span>)</span><br><span class="line">        answer = soup.<span class="builtin-name">find</span>(<span class="string">"body"</span>)</span><br><span class="line"></span><br><span class="line">    <span class="builtin-name">print</span> author_name,answer_id,question_id,question_title,vote_up_count,create_time</span><br><span class="line">    # 这里非原创，看了别人的代码，修改了一下</span><br><span class="line">    soup.body.extract()</span><br><span class="line">    soup.head.insert_after(soup.new_tag(<span class="string">"body"</span>, **&#123;<span class="string">'class'</span>: <span class="string">'zhi'</span>&#125;))</span><br><span class="line"></span><br><span class="line">    soup.body.append(answer)</span><br><span class="line"></span><br><span class="line">    img_list = soup.find_all(<span class="string">"img"</span>, <span class="attribute">class_</span>=<span class="string">"content_image lazy"</span>)</span><br><span class="line">    <span class="keyword">for</span> img <span class="keyword">in</span> img_list:</span><br><span class="line">        img[<span class="string">"src"</span>] = img[<span class="string">"data-actualsrc"</span>]</span><br><span class="line">    img_list = soup.find_all(<span class="string">"img"</span>, <span class="attribute">class_</span>=<span class="string">"origin_image zh-lightbox-thumb lazy"</span>)</span><br><span class="line">    <span class="keyword">for</span> img <span class="keyword">in</span> img_list:</span><br><span class="line">        img[<span class="string">"src"</span>] = img[<span class="string">"data-actualsrc"</span>]</span><br><span class="line">    noscript_list = soup.find_all(<span class="string">"noscript"</span>)</span><br><span class="line">    <span class="keyword">for</span> noscript <span class="keyword">in</span> noscript_list:</span><br><span class="line">        noscript.extract()</span><br><span class="line"></span><br><span class="line">    data[<span class="string">'content'</span>] = soup</span><br><span class="line">    data[<span class="string">'author_name'</span>] = author_name</span><br><span class="line">    data[<span class="string">'answer_id'</span>] = answer_id</span><br><span class="line">    data[<span class="string">'question_id'</span>] = question_id</span><br><span class="line">    data[<span class="string">'question_title'</span>] = question_title</span><br><span class="line">    data[<span class="string">'vote_up_count'</span>] = vote_up_count</span><br><span class="line">    data[<span class="string">'create_time'</span>] = create_time</span><br><span class="line"></span><br><span class="line">    return data</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>parse_content.py主要负责构造新的html，然后对其进行解析，获取数据。</p>
                  <h2 id="5-测试结果展示"><a href="#5-测试结果展示" class="headerlink" title="5.测试结果展示"></a><strong>5.测试结果展示</strong></h2>
                  <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/result.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/result.jpg" alt=""></a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/result_2.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2017/06/result_2.jpg" alt=""></a> 恩，下面还有，就不截图了。</p>
                  <h2 id="6-缺点与不足"><a href="#6-缺点与不足" class="headerlink" title="6.缺点与不足"></a><strong>6.缺点与不足</strong></h2>
                  <p>下面聊一聊这种方法的缺点： 这种方法的最大缺点就是：</p>
                  <h1 id="一定要联网！"><a href="#一定要联网！" class="headerlink" title="一定要联网！"></a>一定要联网！</h1>
                  <h1 id="一定要联网！-1"><a href="#一定要联网！-1" class="headerlink" title="一定要联网！"></a>一定要联网！</h1>
                  <h1 id="一定要联网！-2"><a href="#一定要联网！-2" class="headerlink" title="一定要联网！"></a>一定要联网！</h1>
                  <p>因为。。。。。。 在md文件中我们只是写了个图片的网址，这就意味着markdown的编辑器帮我们去存放图片的服务器上对这个图片进行了获取，所以断网也就意味着你看不到图片了；同时也意味着如果用户删除了这张图片，你也就看不到了。 但是，后来我又发现在markdownpad中将文件导出为html时，即使是断网了，依然可以看到全部的内容，包括图片，所以如果你真的喜欢某一个答案，保存到印象笔记肯定是不错的选择，PDF直接保存也不错，如果是使用了这个方法，记得转为html最好。 还有一个缺点就是html2text转换过后的效果其实并不是特别好，还是需要后期在进行处理的。</p>
                  <h2 id="7-总结"><a href="#7-总结" class="headerlink" title="7.总结"></a><strong>7.总结</strong></h2>
                  <p>代码还有很多可以改进之处，欢迎大家与我交流：QQ:549411552 （注明来自静觅） 国际惯例：<a href="https://github.com/xiaosimao/transfrom_zhihu_answer_to_md" target="_blank" rel="noopener">代码在这</a> 收工。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/四毛" class="author" itemprop="url" rel="index">四毛</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-06-05 23:50:07" itemprop="dateCreated datePublished" datetime="2017-06-05T23:50:07+08:00">2017-06-05</time>
                </span>
                <span id="/4607.html" class="post-meta-item leancloud_visitors" data-flag-title="获取知乎问题答案并转换为MarkDown文件" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>9.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>8 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4596.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4596.html" class="post-title-link" itemprop="url">使用Tornado+Redis维护ADSL拨号服务器代理池</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>我们尝试维护过一个免费的代理池，但是代理池效果用过就知道了，毕竟里面有大量免费代理，虽然这些代理是可用的，但是既然我们能刷到这个免费代理，别人也能呀，所以就导致这个代理同时被很多人使用来抓取网站，所以当我们兴致勃勃地拿他来抓取某个网站的时候，会发现它还是被网站封禁的状态，所以在某些情况下免费代理池的成功率还是比较低的。 当然我们也可以去购买一些代理，比如几块钱提取几百几千个的代理，然而经过测试后质量也是很一般，也可以去购买专线代理，不过价格也是不菲的。那么目前最稳定而且又保证可用的代理方法就是设置ADSL拨号代理了。 本篇来讲解一下ADSL拨号代理服务器的相关设置。</p>
                  <h2 id="什么是ADSL"><a href="#什么是ADSL" class="headerlink" title="什么是ADSL"></a>什么是ADSL</h2>
                  <p>大家可能对ADSL比较陌生，ADSL全称叫做Asymmetric Digital Subscriber Line，非对称数字用户环路，因为它的上行和下行带宽不对称。它采用频分复用技术把普通的电话线分成了电话、上行和下行三个相对独立的信道，从而避免了相互之间的干扰。 有种主机叫做动态拨号VPS主机，这种主机在连接上网的时候是需要拨号的，只有拨号成功后才可以上网，每拨一次号，主机就会获取一个新的IP，也就是它的IP并不是固定的，而且IP量特别大，几乎不会拨到相同的IP，如果我们用它来搭建代理，既能保证高度可用，又可以自由控制拨号切换。 经测试发现这也是最稳定最有效的代理方式，本节详细介绍一下ADSL拨号代理服务器的搭建方法。</p>
                  <h2 id="购买动态拨号VPS主机"><a href="#购买动态拨号VPS主机" class="headerlink" title="购买动态拨号VPS主机"></a>购买动态拨号VPS主机</h2>
                  <p>所以在开始之前，我们需要先购买一台动态拨号VPS主机，这样的主机在百度搜索一下，服务商还是相当多的，在这里推荐一家<a href="http://www.yunlifang.cn/dynamicvps.asp" target="_blank" rel="noopener">云立方</a>，感觉还是比较良心的，非广告。 配置的话可以自行选择，看下带宽是否可以满足需求就好了。 购买完成之后，就需要安装操作系统了，进入拨号主机的后台，首先预装一个操作系统。 <img src="https://blog-10039692.file.myqcloud.com/1495175818199_5646_1495175827700.jpg" alt=""> 在这里推荐安装CentOS7系统。 然后找到远程管理面板找到远程连接的用户名和密码，也就是SSH远程连接服务器的信息。 比如我这边的IP端口分别是 153.36.65.214:20063，用户名是root。 命令行下输入：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ssh <span class="symbol">root@</span><span class="number">153.36</span><span class="number">.65</span><span class="number">.214</span> -p <span class="number">20063</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p> 然后输入管理密码，就可以连接上远程服务器了。 进入之后，可以发现有一个可用的脚本文件，叫做ppp.sh，这是拨号初始化的脚本，运行它会让我们输入拨号的用户名和密码，然后它就会开始各种拨号配置，一次配置成功，后面的拨号就不需要重复输入用户名和密码了。 运行ppp.sh脚本，输入用户名密码等待它的配置完成。 <img src="https://blog-10039692.file.myqcloud.com/1495175987975_6876_1495175998841.jpg" alt=""> 都提示成功之后就可以进行拨号了。 在拨号之前如果我们测试ping任何网站都是不通的，因为当前网络还没联通，输入拨号命令：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">adsl-<span class="literal">start</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现拨号命令成功运行，没有任何报错信息，这就证明拨号成功完成了，耗时约几秒钟。接下来如果再去ping外网就可以通了。 如果要停止拨号可以输入：</p>
                  <figure class="highlight arduino">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">adsl-<span class="built_in">stop</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>停止之后，可以发现又连不通网络了。</p>
                  <p>所以只有拨号之后才可以建立网络连接。 <img src="https://blog-10039692.file.myqcloud.com/1495176020258_290_1495176022867.jpg" alt=""> 所以断线重播的命令就是二者组合起来，先执行<code>adsl-stop</code>再执行<code>adsl-start</code>，每拨一次号，<code>ifocnfig</code>命令观察一下主机的IP，发现主机的IP一直是在变化的，网卡名称叫做ppp0。 <img src="https://blog-10039692.file.myqcloud.com/1495176189060_6947_1495176191282.jpg" alt=""> 所以，到这里我们就可以知道它作为代理服务器的巨大优势了，如果将这台主机作为代理服务器，如果我们一直拨号换IP，就不怕遇到IP被封的情况了，即使某个IP被封了，重新拨一次号就好了。 所以接下来我们要做的就有两件事，一是怎样将主机设置为代理服务器，二是怎样实时获取拨号主机的IP。</p>
                  <h2 id="设置代理服务器"><a href="#设置代理服务器" class="headerlink" title="设置代理服务器"></a>设置代理服务器</h2>
                  <p>之前我们经常听说代理服务器，也设置过不少代理了，但是可能没有自己设置吧，自己有一台主机怎样设置为代理服务器呢？接下来我们就亲自试验下怎样搭建HTTP代理服务器。 在Linux下搭建HTTP代理服务器，推荐TinyProxy和Squid，配置都非常简单，在这里我们以TinyProxy为例来讲解一下怎样搭建代理服务器。</p>
                  <h3 id="安装TinyProxy"><a href="#安装TinyProxy" class="headerlink" title="安装TinyProxy"></a>安装TinyProxy</h3>
                  <p>当然第一步就是安装TinyProxy这个软件了，在这里我使用的系统是CentOS，所以使用yum来安装，如果是其他系统如Ubuntu可以选择apt-get等命令安装，都是类似的。 命令行执行yum安装指令：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">yum <span class="keyword">install</span> -y epel-<span class="keyword">release</span></span><br><span class="line">yum <span class="keyword">update</span> -y</span><br><span class="line">yum <span class="keyword">install</span> -y tinyproxy</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行完成之后就可以完成tinyproxy的安装了。</p>
                  <h3 id="配置TinyProxy"><a href="#配置TinyProxy" class="headerlink" title="配置TinyProxy"></a>配置TinyProxy</h3>
                  <p>安装完成之后还需要配置一下TinyProxy才可以用作代理服务器，需要编辑配置文件，它一般的路径是<code>/etc/tinyproxy/tinyproxy.conf</code>。 可以看到有一行</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Port <span class="number">8888</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里可以设置代理的端口，默认是8888。 然后继续向下找，有这么一行</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Allow <span class="number">127.0</span><span class="number">.0</span><span class="number">.1</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这是被允许连接的主机的IP，如果想任何主机都可以连接，那就直接将它注释即可，所以在这里我们选择直接注释，也就是任何主机都可以使用这台主机作为代理服务器了。 修改为</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Allow <span class="number">127.0</span><span class="number">.0</span><span class="number">.1</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>设置完成之后重启TinyProxy即可。</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">service tinyproxy <span class="literal">start</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>验证TinyProxy 好了，这样我们就成功搭建好代理服务器了，首先<code>ifconfig</code>查看下当前主机的IP，比如当前我的主机拨号IP为<code>112.84.118.216</code>，在其他的主机运行测试一下。 比如用curl命令设置代理请求一下httpbin，检测下代理是否生效。</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl -x <span class="number">112.84</span><span class="number">.118</span><span class="number">.216</span>:<span class="number">8888</span> httpbin.org/<span class="keyword">get</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><img src="https://blog-10039692.file.myqcloud.com/1495176207822_2326_1495176209195.jpg" alt=""> 如果有正常的结果输出并且origin的值为代理IP的地址，就证明TinyProxy配置成功了。 好，那到现在，我们接下来要做的就是需要动态实时获取主机的IP了。</p>
                  <h2 id="动态获取IP"><a href="#动态获取IP" class="headerlink" title="动态获取IP"></a>动态获取IP</h2>
                  <p>真正的好戏才开始呢，我们怎样动态获取主机的IP呢？可能你首先想到的是DDNS也就是动态域名解析服务，我们需要使用一个域名来解析，也就是虽然IP是变的，但域名解析的地址可以随着IP的变化而变化。 它的原理其实是拨号主机向固定的服务器发出请求，服务器获取客户端的IP，然后再将域名解析到这个IP上就可以了。 国内比较有名的服务就是<a href="http://hsk.oray.com/" target="_blank" rel="noopener">花生壳</a>了，也提供了免费版的动态域名解析，另外DNSPOD也提供了解析接口来动态修改域名解析设置，<a href="https://www.dnspod.cn/docs/records.html#dns" target="_blank" rel="noopener">DNSPOD</a>，但是这样的方式都有一个通病，那就是慢！ 原因在于DNS修改后到完全生效是需要一定时间的，所以如果在前一秒拨号了，这一秒的域名解析的可能还是原来的IP，时间长的话可能需要几分钟，也就是说这段时间内，服务器IP已经变了，但是域名还是上一次拨号的IP，所以代理是不能用的，对于爬虫这种秒级响应的需求，是完全不能接受的。 所以根据花生壳的原理，可以完全自己实现一下动态获取IP的方法。 所以本节重点介绍的就是怎样来实现实时获取拨号主机IP的方法。 要实现这个需要两台主机，一台主机就是这台动态拨号VPS主机，另一台是具有固定公网IP的主机。动态VPS主机拨号成功之后就请求远程的固定主机，远程主机获取动态VPS主机的IP，就可以得到这个代理，将代理保存下来，这样拨号主机每拨号一次，远程主机就会及时得到拨号主机的IP，如果有多台拨号VPS，也统一发送到远程主机，这样我们只需要从远程主机取下代理就好了，保准是实时可用，稳定高效的。 整体思路大体是这样子，当然为了更完善一下，我们要做到如下功能： 远程主机：</p>
                  <ul>
                    <li>监听主机请求，获取动态VPS主机IP</li>
                    <li>将VPS主机IP记录下来存入数据库，支持多个客户端</li>
                    <li>检测当前接收到的IP可用情况，如果不可用则删除</li>
                    <li>提供API接口，通过API接口可获取当前可用代理IP</li>
                  </ul>
                  <p>拨号VPS：</p>
                  <ul>
                    <li>定时执行拨号脚本换IP</li>
                    <li>换IP后立即请求远程主机</li>
                    <li>拨号后检测是否拨号成功，如果失败立即重新拨号</li>
                  </ul>
                  <h3 id="远程主机实现"><a href="#远程主机实现" class="headerlink" title="远程主机实现"></a>远程主机实现</h3>
                  <p>说了这么多，那么我们就梳理一下具体的实现吧，整个项目我们用Python3实现。</p>
                  <h4 id="数据库"><a href="#数据库" class="headerlink" title="数据库"></a>数据库</h4>
                  <p>远程主机作为一台服务器，动态拨号VPS会定时请求远程主机，远程主机接收到请求后将IP记录下来存入数据库。 因为IP是一直在变化的，IP更新了之后，原来的IP就不能用了，所以对于一个主机来说我们可能需要多次更新一条数据。另外我们不能仅限于维护一台拨号VPS主机，当然是需要支持多台维护的。在这里我们直接选用Key-Value形式的非关系型数据库存储更加方便，所以在此选用Redis数据库。 既然是Key-Value，Key是什么?Value是什么?首先我们能确定Value就是代理的值，比如112.84.119.67:8888，那么Key是什么？我们知道，这个IP是针对一台动态拨号VPS的，而且这个值会不断地变，所以我们需要有一个不变量Key来唯一标识这台主机，所以在这里我们可以把Key当做主机名称。名称怎么来？自己取就好了，只要每台主机的名字不重复，我们就可以区分出是哪台主机了，这个名字可以在拨号主机那边指定，然后传给远程主机就好了。 所以，在这里数据库我们选用Redis，Key就是拨号主机的名称，可以自己指定，Value就是代理的值。 所以可以写一个操作Redis数据库的类，参考如下：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">RedisClient</span>(<span class="title">object</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, host=REDIS_HOST, port=REDIS_PORT)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.db = redis.Redis(host=host, port=port, password=REDIS_PASSWORD)</span><br><span class="line">        <span class="keyword">self</span>.proxy_key = PROXY_KEY</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">key</span><span class="params">(<span class="keyword">self</span>, name)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> <span class="string">'&#123;key&#125;:&#123;name&#125;'</span>.format(key=<span class="keyword">self</span>.proxy_key, name=name)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">set</span><span class="params">(<span class="keyword">self</span>, name, proxy)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">self</span>.db.set(<span class="keyword">self</span>.key(name), proxy)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get</span><span class="params">(<span class="keyword">self</span>, name)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">self</span>.db.get(<span class="keyword">self</span>.key(name)).decode(<span class="string">'utf-8'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先初始化Redis连接，我们可以将Key设计成<code>adsl:vm1</code>这种形式，冒号前面是总的key，冒号后面是主机名称name，这样显得结构更加清晰。 然后指定set()和get()方法，用来存储代理和获取代理。</p>
                  <h4 id="请求处理"><a href="#请求处理" class="headerlink" title="请求处理"></a>请求处理</h4>
                  <p>拨号主机会一直向远程主机发送请求，远程主机当然可以获取拨号主机的IP，但是代理端口是无法获得的，我们在拨号主机上设置了TinyProxy或者Squid，但是服务器不知道是在哪个端口开的，所以端口也是需要客户端传给远程主机的。远程主机接收到请求后，将解析得到的IP和端口合并就可以作为完整的代理保存了。 所以现在我们知道拨号主机需要传送给远程主机的信息已经有两个了，一是拨号主机本身的名称，二是代理的端口。</p>
                  <h4 id="通信秘钥"><a href="#通信秘钥" class="headerlink" title="通信秘钥"></a>通信秘钥</h4>
                  <p>为了保证远程主机不被恶意的请求干扰，可以设置一个传输秘钥，最简单的方式可以二者共同规定一个秘钥字符串，拨号主机在传送这个字符串，远程主机匹配一下，如果能正确匹配，那就进行下一步的处理，如果不能匹配，那么可能是恶意请求，就忽略这个请求。 当然肯定有更好的加密传输方式，但为了方便起见可以用如上来做。 所以客户机还需要传送一个数据，那就是通信秘钥，一共需要传送三个数据。 所以我们需要架设一个服务器，一直监听客户端的请求，在这里我们用tornado实现。 tornado的安装也非常简单，利用pip安装即可：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> tornado</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>定义一个处理拨号主机请求的方法，在这里我们使用post请求，参考如下。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def post(self):</span><br><span class="line">        token = self.get_body_argument(<span class="string">'token'</span>, <span class="attribute">default</span>=None, <span class="attribute">strip</span>=<span class="literal">False</span>)</span><br><span class="line">       <span class="built_in"> port </span>= self.get_body_argument(<span class="string">'port'</span>, <span class="attribute">default</span>=None, <span class="attribute">strip</span>=<span class="literal">False</span>)</span><br><span class="line">        name = self.get_body_argument(<span class="string">'name'</span>, <span class="attribute">default</span>=None, <span class="attribute">strip</span>=<span class="literal">False</span>)</span><br><span class="line">        <span class="keyword">if</span> token == TOKEN <span class="keyword">and</span> port:</span><br><span class="line">           <span class="built_in"> ip </span>= self.request.remote_ip</span><br><span class="line">           <span class="built_in"> proxy </span>=<span class="built_in"> ip </span>+ <span class="string">':'</span> + port</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'Receive proxy'</span>, proxy)</span><br><span class="line">            self.redis.<span class="builtin-name">set</span>(name, proxy)</span><br><span class="line">            self.test_proxies()</span><br><span class="line">        elif token != TOKEN:</span><br><span class="line">            self.write(<span class="string">'Wrong Token'</span>)</span><br><span class="line">        elif <span class="keyword">not</span> port:</span><br><span class="line">            self.write(<span class="string">'No Client Port'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>远程主机获取请求的token，也就是上面我们所说的通信密钥，保证安全。port是拨号机的代理端口，name是拨号主机的名称。然后我们再获取请求的remote_ip，也就是拨号主机的IP。然后将IP和端口拼合就可以得到拨号主机的完整代理信息了，将其存入数据库即可。</p>
                  <h4 id="代理检测"><a href="#代理检测" class="headerlink" title="代理检测"></a>代理检测</h4>
                  <p>在远程主机端我们需要做一下代理检测，如果某个代理不可用了，会及时将其去除，以免出现获取到代理后不可用的情况。</p>
                  <blockquote>
                    <p>注意：在这里在拨号主机端验证是不够的，因为可能突然遇到某个拨号主机宕机的情况，这样拨号主机就不会再向远程主机发送请求，而最后一次得到的代理还会存在于数据库中，所以在远程主机端统一验证比较科学。</p>
                  </blockquote>
                  <p>验证方式可以定时检测，也可以每收到一次请求检测一次，用获取到的代理来请求某个网站，检测一下是否能访问即可。如果不能，将其从数据库中删除。</p>
                  <h4 id="API"><a href="#API" class="headerlink" title="API"></a>API</h4>
                  <p>远程主机已经将拨号主机的IP和端口保存下来了，那也就是说，所有的可用的代理已经在远程主机保存了，我们需要提供一个接口来将代理获取下来。 比如我们可以提供这么几个方法，获取所有代理，获取最新代理，获取随机代理等等。</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">all</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    keys = <span class="keyword">self</span>.keys()</span><br><span class="line">    proxies = [&#123;<span class="string">'name'</span>: key, <span class="string">'proxy'</span>: <span class="keyword">self</span>.get(key)&#125; <span class="keyword">for</span> key <span class="keyword">in</span> keys]</span><br><span class="line">    <span class="keyword">return</span> proxies</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">random</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    items = <span class="keyword">self</span>.all()</span><br><span class="line">    <span class="keyword">return</span> random.choice(items).get(<span class="string">'proxy'</span>)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">list</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    keys = <span class="keyword">self</span>.keys()</span><br><span class="line">    proxies = [<span class="keyword">self</span>.get(key) <span class="keyword">for</span> key <span class="keyword">in</span> keys]</span><br><span class="line">    <span class="keyword">return</span> proxies</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">first</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">    <span class="keyword">return</span> <span class="keyword">self</span>.get(<span class="keyword">self</span>.keys()[<span class="number">0</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后用tornado搭建API服务，如果可以的话还可以绑定一个域名，更加便捷，举例如下： 获取随机代理： <img src="https://blog-10039692.file.myqcloud.com/1495176234257_8333_1495176237128.jpg" alt=""> 获取最新代理： <img src="https://blog-10039692.file.myqcloud.com/1495176256328_2908_1495176259312.jpg" alt=""> 获取所有代理： <img src="https://blog-10039692.file.myqcloud.com/1495176279029_802_1495176279909.jpg" alt=""> 请求接口获取可用代理即可，比如获取一个随机代理：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_random_proxy</span><span class="params">()</span>:</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        <span class="comment"># 远程主机的服务地址</span></span><br><span class="line">        url = <span class="string">'http://xxx.xxx.xxx.xxx:8000/random'</span></span><br><span class="line">        <span class="keyword">return</span> requests.get(url).text</span><br><span class="line">    <span class="keyword">except</span> requests.exceptions.ConnectionError:</span><br><span class="line">        <span class="keyword">return</span> <span class="literal">None</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们拿到的IP都是稳定可用的，而且过段时间重新请求取到的IP就会变化，是一直动态变化的高可用代理。</p>
                  <h3 id="拨号VPS实现"><a href="#拨号VPS实现" class="headerlink" title="拨号VPS实现"></a>拨号VPS实现</h3>
                  <h4 id="定时拨号"><a href="#定时拨号" class="headerlink" title="定时拨号"></a>定时拨号</h4>
                  <p>拨号VPS需要每隔一段时间就拨号一次，我们可以直接执行命令行来拨号，那在Python里我们只需要调用一下这个拨号命令就好了。利用subprocess模块调用脚本即可，在这里定义一个变量ADSL_BASH为<code>adsl-stop;adsl-start</code>，这就是拨号的脚本。</p>
                  <figure class="highlight cpp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> subprocess</span><br><span class="line">(status, output) = subprocess.getstatusoutput(ADSL_BASH)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过getstatusoutput方法可以获取脚本的执行状态和输出结果，如果status为0，则证明拨号成功，然后检测一下拨号接口是否获取了IP地址。 执行<code>ifconfig</code>命令可以获取当前的IP，我这台主机接口名称叫做ppp0，当然网卡名称可以自己指定，所以将ppp0接口的IP提取出来即可。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def get_ip(self, <span class="attribute">ifname</span>=ADSL_IFNAME):</span><br><span class="line">    (status, output) = subprocess.getstatusoutput(<span class="string">'ifconfig'</span>)</span><br><span class="line">    <span class="keyword">if</span> status == 0:</span><br><span class="line">        pattern = re.compile(ifname + <span class="string">'.*?inet.*?(\d+\.\d+\.\d+\.\d+).*?netmask'</span>, re.S)</span><br><span class="line">        result = re.search(pattern, output)</span><br><span class="line">        <span class="keyword">if</span> result:</span><br><span class="line">           <span class="built_in"> ip </span>= result.group(1)</span><br><span class="line">            return ip</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果方法正常返回IP，则证明IP存在，拨号成功，接下来向远程主机发送请求即可，然后sleep一段时间重新再次拨号。 如果方法返回的值为空，那证明IP不存在，我们需要重新拨号。</p>
                  <h4 id="请求远程主机"><a href="#请求远程主机" class="headerlink" title="请求远程主机"></a>请求远程主机</h4>
                  <p>发送的时候需要携带这么几个信息，一个是通信秘钥，一个是代理端口，另一个是主机的标识符，用requests发送即可。</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="title">requests</span>.post(<span class="type">SERVER_URL</span>, <span class="class"><span class="keyword">data</span>=&#123;'<span class="title">token'</span>: <span class="type">TOKEN</span>, '<span class="title">port'</span>: <span class="type">PROXY_PORT</span>, '<span class="title">name'</span>: <span class="type">CLIENT_NAME</span>&#125;)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>所以整体的思路实现可以写成这样子：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def adsl(self):</span><br><span class="line">        <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'ADSL Start, Please wait'</span>)</span><br><span class="line">            (status, output) = subprocess.getstatusoutput(ADSL_BASH)</span><br><span class="line">            <span class="keyword">if</span> status == 0:</span><br><span class="line">                <span class="builtin-name">print</span>(<span class="string">'ADSL Successfully'</span>)</span><br><span class="line">               <span class="built_in"> ip </span>= self.get_ip()</span><br><span class="line">                <span class="keyword">if</span> ip:</span><br><span class="line">                    <span class="builtin-name">print</span>(<span class="string">'New IP'</span>, ip)</span><br><span class="line">                    try:</span><br><span class="line">                        requests.post(SERVER_URL, data=&#123;<span class="string">'token'</span>: TOKEN, <span class="string">'port'</span>: PROXY_PORT, <span class="string">'name'</span>: CLIENT_NAME&#125;)</span><br><span class="line">                        <span class="builtin-name">print</span>(<span class="string">'Successfully Sent to Server'</span>, SERVER_URL)</span><br><span class="line">                    except ConnectionError:</span><br><span class="line">                        <span class="builtin-name">print</span>(<span class="string">'Failed to Connect Server'</span>, SERVER_URL)</span><br><span class="line">                    time.sleep(ADSL_CYCLE)</span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    <span class="builtin-name">print</span>(<span class="string">'Get IP Failed'</span>)</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                <span class="builtin-name">print</span>(<span class="string">'ADSL Failed, Please Check'</span>)</span><br><span class="line">            time.sleep(1)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就可以做到定时拨号并向远程主机发送请求了。</p>
                  <h2 id="代码"><a href="#代码" class="headerlink" title="代码"></a>代码</h2>
                  <p>Talk is cheap, show me the code! 在这里提供一份完整代码实现，其中client模块是在动态VPS主机运行，server模块在远程主机运行，具体的操作使用可以参考README。 <a href="https://github.com/Germey/ADSLProxyPool" target="_blank" rel="noopener">ADSLProxyPool</a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-05-19 14:50:15" itemprop="dateCreated datePublished" datetime="2017-05-19T14:50:15+08:00">2017-05-19</time>
                </span>
                <span id="/4596.html" class="post-meta-item leancloud_visitors" data-flag-title="使用Tornado+Redis维护ADSL拨号服务器代理池" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>7.8k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4534.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4534.html" class="post-title-link" itemprop="url">Scrapyd日志输出优化</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>现在维护着一个新浪微博爬虫，爬取量已经5亿+，使用了Scrapyd部署分布式。 Scrapyd运行时会输出日志到本地，导致日志文件会越来越大，这个其实就是Scrapy控制台的输出。但是这个日志其实有用的部分也就是最后那几百行而已，如果出错，去日志查看下出错信息就好了。 所以现在可以写一个脚本，来定时更新日志文件，将最后的100行保存下来就好了。 Scrapyd默认的日志目录是在用户文件夹下的logs目录。 所以在这里我们指定dir=~/logs 新建bash脚本，内容如下：</p>
                  <figure class="highlight bash">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">#!/bin/sh</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="title">clean</span></span>() &#123;</span><br><span class="line">  <span class="keyword">for</span> file <span class="keyword">in</span> <span class="variable">$1</span>/*</span><br><span class="line">  <span class="keyword">do</span></span><br><span class="line">    <span class="keyword">if</span> [ -d <span class="variable">$file</span> ]</span><br><span class="line">    <span class="keyword">then</span></span><br><span class="line">      clean <span class="variable">$file</span></span><br><span class="line">    <span class="keyword">else</span></span><br><span class="line">      <span class="built_in">echo</span> <span class="variable">$file</span></span><br><span class="line">      temp=$(tail -100 <span class="variable">$file</span>)</span><br><span class="line">      <span class="built_in">echo</span> <span class="string">"<span class="variable">$temp</span>"</span> &gt; <span class="variable">$file</span></span><br><span class="line">    <span class="keyword">fi</span></span><br><span class="line">  <span class="keyword">done</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">dir=~/logs</span><br><span class="line">clean <span class="variable">$dir</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>新建这样的一个脚本，然后命名为 clean.sh，我的直接放在了用户文件夹下。 然后crontab创建定时任务。 执行</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">crontab -e</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们想要一分钟清理一次日志文件。 输入</p>
                  <figure class="highlight jboss-cli">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">*<span class="string">/1</span> * * * * <span class="string">/bin/sh</span> ~<span class="string">/clean.sh</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后退出之后，crontab就可以每隔一分钟执行一次clean.sh，清理日志了。 这样我们就不怕日志文件大量占用主机空间啦~</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2017-05-17 14:49:03" itemprop="dateCreated datePublished" datetime="2017-05-17T14:49:03+08:00">2017-05-17</time>
                </span>
                <span id="/4534.html" class="post-meta-item leancloud_visitors" data-flag-title="Scrapyd日志输出优化" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>579</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <script>
              document.querySelectorAll('.random').forEach(item => item.src = "https://picsum.photos/id/" + Math.floor(Math.random() * Math.floor(300)) + "/200/133")

            </script>
            <nav class="pagination">
              <a class="extend prev" rel="prev" href="/page/20/"><i class="fa fa-angle-left" aria-label="上一页"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/20/">20</a><span class="page-number current">21</span><a class="page-number" href="/page/22/">22</a><span class="space">&hellip;</span><a class="page-number" href="/page/31/">31</a><a class="extend next" rel="next" href="/page/22/"><i class="fa fa-angle-right" aria-label="下一页"></i></a>
            </nav>
          </div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
</body>

</html>
